class MyAnalyzer extends Analyzer {
{@literal @Override}
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer source = new LowerCaseTokenizer(version, reader);
return new TokenStreamComponents(source, new PorterStemFilter(source));
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java (working copy)
@@ -116,9 +116,8 @@
* provided and {@link SpanishLightStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (working copy)
@@ -114,9 +114,8 @@
* {@link PersianNormalizationFilter} and Persian Stop words
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
result = new ArabicNormalizationFilter(result);
/* additional persian-specific normalization */
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java (working copy)
@@ -117,9 +117,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (working copy)
@@ -137,9 +137,8 @@
* provided, and {@link FrenchLightStemFilter}
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java (working copy)
@@ -133,9 +133,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new StopFilter(matchVersion, result, HYPHENATIONS);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* provided and {@link GalicianStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiAnalyzer.java (working copy)
@@ -116,9 +116,8 @@
* Hindi Stop words
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new LowerCaseFilter(matchVersion, source);
if (!stemExclusionSet.isEmpty())
result = new SetKeywordMarkerFilter(result, stemExclusionSet);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java (working copy)
@@ -117,9 +117,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/id/IndonesianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/id/IndonesianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/id/IndonesianAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* if a stem exclusion set is provided and {@link IndonesianStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java (working copy)
@@ -126,9 +126,8 @@
* provided and {@link ItalianLightStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* provided and {@link LatvianStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (working copy)
@@ -38,12 +38,11 @@
* Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range
*
* @param version the Lucene match version
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public EdgeNGramTokenizer(Version version, Reader input, int minGram, int maxGram) {
- super(version, input, minGram, maxGram, true);
+ public EdgeNGramTokenizer(Version version, int minGram, int maxGram) {
+ super(version, minGram, maxGram, true);
}
/**
@@ -51,12 +50,11 @@
*
* @param version the Lucene match version
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public EdgeNGramTokenizer(Version version, AttributeFactory factory, Reader input, int minGram, int maxGram) {
- super(version, factory, input, minGram, maxGram, true);
+ public EdgeNGramTokenizer(Version version, AttributeFactory factory, int minGram, int maxGram) {
+ super(version, factory, minGram, maxGram, true);
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (working copy)
@@ -47,7 +47,7 @@
}
@Override
- public EdgeNGramTokenizer create(AttributeFactory factory, Reader input) {
- return new EdgeNGramTokenizer(luceneMatchVersion, factory, input, minGramSize, maxGramSize);
+ public EdgeNGramTokenizer create(AttributeFactory factory) {
+ return new EdgeNGramTokenizer(luceneMatchVersion, factory, minGramSize, maxGramSize);
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/Lucene43NGramTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/Lucene43NGramTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/Lucene43NGramTokenizer.java (working copy)
@@ -45,33 +45,29 @@
/**
* Creates NGramTokenizer with given min and max n-grams.
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public Lucene43NGramTokenizer(Reader input, int minGram, int maxGram) {
- super(input);
+ public Lucene43NGramTokenizer(int minGram, int maxGram) {
init(minGram, maxGram);
}
/**
* Creates NGramTokenizer with given min and max n-grams.
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public Lucene43NGramTokenizer(AttributeFactory factory, Reader input, int minGram, int maxGram) {
- super(factory, input);
+ public Lucene43NGramTokenizer(AttributeFactory factory, int minGram, int maxGram) {
+ super(factory);
init(minGram, maxGram);
}
/**
* Creates NGramTokenizer with default min and max n-grams.
- * @param input {@link Reader} holding the input to be tokenized
*/
- public Lucene43NGramTokenizer(Reader input) {
- this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
+ public Lucene43NGramTokenizer() {
+ this(DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
}
private void init(int minGram, int maxGram) {
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (working copy)
@@ -77,24 +77,22 @@
private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- NGramTokenizer(Version version, Reader input, int minGram, int maxGram, boolean edgesOnly) {
- super(input);
+ NGramTokenizer(Version version, int minGram, int maxGram, boolean edgesOnly) {
init(version, minGram, maxGram, edgesOnly);
}
/**
* Creates NGramTokenizer with given min and max n-grams.
* @param version the lucene compatibility version
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public NGramTokenizer(Version version, Reader input, int minGram, int maxGram) {
- this(version, input, minGram, maxGram, false);
+ public NGramTokenizer(Version version, int minGram, int maxGram) {
+ this(version, minGram, maxGram, false);
}
- NGramTokenizer(Version version, AttributeFactory factory, Reader input, int minGram, int maxGram, boolean edgesOnly) {
- super(factory, input);
+ NGramTokenizer(Version version, AttributeFactory factory, int minGram, int maxGram, boolean edgesOnly) {
+ super(factory);
init(version, minGram, maxGram, edgesOnly);
}
@@ -102,21 +100,19 @@
* Creates NGramTokenizer with given min and max n-grams.
* @param version the lucene compatibility version
* @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use
- * @param input {@link Reader} holding the input to be tokenized
* @param minGram the smallest n-gram to generate
* @param maxGram the largest n-gram to generate
*/
- public NGramTokenizer(Version version, AttributeFactory factory, Reader input, int minGram, int maxGram) {
- this(version, factory, input, minGram, maxGram, false);
+ public NGramTokenizer(Version version, AttributeFactory factory, int minGram, int maxGram) {
+ this(version, factory, minGram, maxGram, false);
}
/**
* Creates NGramTokenizer with default min and max n-grams.
* @param version the lucene compatibility version
- * @param input {@link Reader} holding the input to be tokenized
*/
- public NGramTokenizer(Version version, Reader input) {
- this(version, input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
+ public NGramTokenizer(Version version) {
+ this(version, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE);
}
private void init(Version version, int minGram, int maxGram, boolean edgesOnly) {
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (working copy)
@@ -51,11 +51,11 @@
/** Creates the {@link TokenStream} of n-grams from the given {@link Reader} and {@link AttributeFactory}. */
@Override
- public Tokenizer create(AttributeFactory factory, Reader input) {
+ public Tokenizer create(AttributeFactory factory) {
if (luceneMatchVersion.onOrAfter(Version.LUCENE_44)) {
- return new NGramTokenizer(luceneMatchVersion, factory, input, minGramSize, maxGramSize);
+ return new NGramTokenizer(luceneMatchVersion, factory, minGramSize, maxGramSize);
} else {
- return new Lucene43NGramTokenizer(factory, input, minGramSize, maxGramSize);
+ return new Lucene43NGramTokenizer(factory, minGramSize, maxGramSize);
}
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (working copy)
@@ -155,9 +155,8 @@
* {@link StemmerOverrideFilter}, and {@link SnowballFilter}
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader aReader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, aReader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stoptable);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java (working copy)
@@ -117,9 +117,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java (working copy)
@@ -43,37 +43,37 @@
*/
public class PathHierarchyTokenizer extends Tokenizer {
- public PathHierarchyTokenizer(Reader input) {
- this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
+ public PathHierarchyTokenizer() {
+ this( DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
}
- public PathHierarchyTokenizer(Reader input, int skip) {
- this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, skip);
+ public PathHierarchyTokenizer( int skip) {
+ this( DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, skip);
}
- public PathHierarchyTokenizer(Reader input, int bufferSize, char delimiter) {
- this(input, bufferSize, delimiter, delimiter, DEFAULT_SKIP);
+ public PathHierarchyTokenizer( int bufferSize, char delimiter) {
+ this( bufferSize, delimiter, delimiter, DEFAULT_SKIP);
}
- public PathHierarchyTokenizer(Reader input, char delimiter, char replacement) {
- this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, DEFAULT_SKIP);
+ public PathHierarchyTokenizer(char delimiter, char replacement) {
+ this(DEFAULT_BUFFER_SIZE, delimiter, replacement, DEFAULT_SKIP);
}
- public PathHierarchyTokenizer(Reader input, char delimiter, char replacement, int skip) {
- this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
+ public PathHierarchyTokenizer(char delimiter, char replacement, int skip) {
+ this(DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
}
- public PathHierarchyTokenizer(AttributeFactory factory, Reader input, char delimiter, char replacement, int skip) {
- this(factory, input, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
+ public PathHierarchyTokenizer(AttributeFactory factory, char delimiter, char replacement, int skip) {
+ this(factory, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
}
- public PathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement, int skip) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, bufferSize, delimiter, replacement, skip);
+ public PathHierarchyTokenizer(int bufferSize, char delimiter, char replacement, int skip) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, bufferSize, delimiter, replacement, skip);
}
public PathHierarchyTokenizer
- (AttributeFactory factory, Reader input, int bufferSize, char delimiter, char replacement, int skip) {
- super(factory, input);
+ (AttributeFactory factory, int bufferSize, char delimiter, char replacement, int skip) {
+ super(factory);
if (bufferSize < 0) {
throw new IllegalArgumentException("bufferSize cannot be negative");
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java (working copy)
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import java.io.Reader;
import java.util.Map;
import org.apache.lucene.analysis.Tokenizer;
@@ -87,11 +86,11 @@
}
@Override
- public Tokenizer create(AttributeFactory factory, Reader input) {
+ public Tokenizer create(AttributeFactory factory) {
if (reverse) {
- return new ReversePathHierarchyTokenizer(factory, input, delimiter, replacement, skip);
+ return new ReversePathHierarchyTokenizer(factory, delimiter, replacement, skip);
}
- return new PathHierarchyTokenizer(factory, input, delimiter, replacement, skip);
+ return new PathHierarchyTokenizer(factory, delimiter, replacement, skip);
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/path/ReversePathHierarchyTokenizer.java (working copy)
@@ -47,45 +47,45 @@
*/
public class ReversePathHierarchyTokenizer extends Tokenizer {
- public ReversePathHierarchyTokenizer(Reader input) {
- this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
+ public ReversePathHierarchyTokenizer() {
+ this(DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
}
- public ReversePathHierarchyTokenizer(Reader input, int skip) {
- this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, skip);
+ public ReversePathHierarchyTokenizer(int skip) {
+ this(DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, skip);
}
- public ReversePathHierarchyTokenizer(Reader input, int bufferSize, char delimiter) {
- this(input, bufferSize, delimiter, delimiter, DEFAULT_SKIP);
+ public ReversePathHierarchyTokenizer(int bufferSize, char delimiter) {
+ this(bufferSize, delimiter, delimiter, DEFAULT_SKIP);
}
- public ReversePathHierarchyTokenizer(Reader input, char delimiter, char replacement) {
- this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, DEFAULT_SKIP);
+ public ReversePathHierarchyTokenizer(char delimiter, char replacement) {
+ this(DEFAULT_BUFFER_SIZE, delimiter, replacement, DEFAULT_SKIP);
}
- public ReversePathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement) {
- this(input, bufferSize, delimiter, replacement, DEFAULT_SKIP);
+ public ReversePathHierarchyTokenizer(int bufferSize, char delimiter, char replacement) {
+ this(bufferSize, delimiter, replacement, DEFAULT_SKIP);
}
- public ReversePathHierarchyTokenizer(Reader input, char delimiter, int skip) {
- this(input, DEFAULT_BUFFER_SIZE, delimiter, delimiter, skip);
+ public ReversePathHierarchyTokenizer(char delimiter, int skip) {
+ this( DEFAULT_BUFFER_SIZE, delimiter, delimiter, skip);
}
- public ReversePathHierarchyTokenizer(Reader input, char delimiter, char replacement, int skip) {
- this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
+ public ReversePathHierarchyTokenizer(char delimiter, char replacement, int skip) {
+ this(DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
}
public ReversePathHierarchyTokenizer
- (AttributeFactory factory, Reader input, char delimiter, char replacement, int skip) {
- this(factory, input, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
+ (AttributeFactory factory, char delimiter, char replacement, int skip) {
+ this(factory, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
}
- public ReversePathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement, int skip) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, bufferSize, delimiter, replacement, skip);
+ public ReversePathHierarchyTokenizer( int bufferSize, char delimiter, char replacement, int skip) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, bufferSize, delimiter, replacement, skip);
}
public ReversePathHierarchyTokenizer
- (AttributeFactory factory, Reader input, int bufferSize, char delimiter, char replacement, int skip) {
- super(factory, input);
+ (AttributeFactory factory, int bufferSize, char delimiter, char replacement, int skip) {
+ super(factory);
if (bufferSize < 0) {
throw new IllegalArgumentException("bufferSize cannot be negative");
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (working copy)
@@ -64,13 +64,13 @@
private final Matcher matcher;
/** creates a new PatternTokenizer returning tokens from group (-1 for split functionality) */
- public PatternTokenizer(Reader input, Pattern pattern, int group) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, pattern, group);
+ public PatternTokenizer(Pattern pattern, int group) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, pattern, group);
}
/** creates a new PatternTokenizer returning tokens from group (-1 for split functionality) */
- public PatternTokenizer(AttributeFactory factory, Reader input, Pattern pattern, int group) {
- super(factory, input);
+ public PatternTokenizer(AttributeFactory factory, Pattern pattern, int group) {
+ super(factory);
this.group = group;
// Use "" instead of str so don't consume chars
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java (working copy)
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import java.io.Reader;
import java.util.Map;
import java.util.regex.Pattern;
@@ -81,7 +80,7 @@
* Split the input using configured pattern
*/
@Override
- public PatternTokenizer create(final AttributeFactory factory, final Reader in) {
- return new PatternTokenizer(factory, in, pattern, group);
+ public PatternTokenizer create(final AttributeFactory factory) {
+ return new PatternTokenizer(factory, pattern, group);
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java (working copy)
@@ -116,9 +116,8 @@
* provided and {@link PortugueseLightStemFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ro/RomanianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ro/RomanianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ro/RomanianAnalyzer.java (working copy)
@@ -120,9 +120,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (working copy)
@@ -115,9 +115,8 @@
* provided, and {@link SnowballFilter}
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (working copy)
@@ -106,8 +106,8 @@
}
@Override
- protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
- final ClassicTokenizer src = new ClassicTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(final String fieldName) {
+ final ClassicTokenizer src = new ClassicTokenizer(matchVersion);
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new ClassicFilter(src);
tok = new LowerCaseFilter(matchVersion, tok);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (working copy)
@@ -96,20 +96,17 @@
* Creates a new instance of the {@link ClassicTokenizer}. Attaches
* the input to the newly created JFlex scanner.
*
- * @param input The input reader
- *
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
- public ClassicTokenizer(Version matchVersion, Reader input) {
- super(input);
+ public ClassicTokenizer(Version matchVersion) {
init(matchVersion);
}
/**
* Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
- public ClassicTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
- super(factory, input);
+ public ClassicTokenizer(Version matchVersion, AttributeFactory factory) {
+ super(factory);
init(matchVersion);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java (working copy)
@@ -20,7 +20,6 @@
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
-import java.io.Reader;
import java.util.Map;
/**
@@ -46,8 +45,8 @@
}
@Override
- public ClassicTokenizer create(AttributeFactory factory, Reader input) {
- ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory, input);
+ public ClassicTokenizer create(AttributeFactory factory) {
+ ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (working copy)
@@ -107,8 +107,8 @@
}
@Override
- protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
- final StandardTokenizer src = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(final String fieldName) {
+ final StandardTokenizer src = new StandardTokenizer(matchVersion);
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
tok = new LowerCaseFilter(matchVersion, tok);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy)
@@ -109,25 +109,22 @@
/**
* Creates a new instance of the {@link org.apache.lucene.analysis.standard.StandardTokenizer}. Attaches
* the input to the newly created JFlex scanner.
- *
- * @param input The input reader
- *
+
* See http://issues.apache.org/jira/browse/LUCENE-1068
*/
- public StandardTokenizer(Version matchVersion, Reader input) {
- super(input);
+ public StandardTokenizer(Version matchVersion) {
init(matchVersion);
}
/**
* Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
*/
- public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
- super(factory, input);
+ public StandardTokenizer(Version matchVersion, AttributeFactory factory) {
+ super(factory);
init(matchVersion);
}
- private final void init(Version matchVersion) {
+ private void init(Version matchVersion) {
this.scanner = new StandardTokenizerImpl(input);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java (working copy)
@@ -20,7 +20,6 @@
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
-import java.io.Reader;
import java.util.Map;
/**
@@ -46,8 +45,8 @@
}
@Override
- public StandardTokenizer create(AttributeFactory factory, Reader input) {
- StandardTokenizer tokenizer = new StandardTokenizer(luceneMatchVersion, factory, input);
+ public StandardTokenizer create(AttributeFactory factory) {
+ StandardTokenizer tokenizer = new StandardTokenizer(luceneMatchVersion, factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailAnalyzer.java (working copy)
@@ -96,8 +96,8 @@
}
@Override
- protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
- final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(final String fieldName) {
+ final UAX29URLEmailTokenizer src = new UAX29URLEmailTokenizer(matchVersion);
src.setMaxTokenLength(maxTokenLength);
TokenStream tok = new StandardFilter(matchVersion, src);
tok = new LowerCaseFilter(matchVersion, tok);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (working copy)
@@ -95,19 +95,17 @@
/**
* Creates a new instance of the UAX29URLEmailTokenizer. Attaches
* the input to the newly created JFlex scanner.
- *
- * @param input The input reader
+
*/
- public UAX29URLEmailTokenizer(Version matchVersion, Reader input) {
- super(input);
+ public UAX29URLEmailTokenizer(Version matchVersion) {
this.scanner = getScannerFor(matchVersion);
}
/**
* Creates a new UAX29URLEmailTokenizer with a given {@link AttributeFactory}
*/
- public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
- super(factory, input);
+ public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory) {
+ super(factory);
this.scanner = getScannerFor(matchVersion);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (working copy)
@@ -46,8 +46,8 @@
}
@Override
- public UAX29URLEmailTokenizer create(AttributeFactory factory, Reader input) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory, input);
+ public UAX29URLEmailTokenizer create(AttributeFactory factory) {
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory);
tokenizer.setMaxTokenLength(maxTokenLength);
return tokenizer;
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java (working copy)
@@ -117,9 +117,8 @@
* provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java (working copy)
@@ -132,8 +132,8 @@
} else {
analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_CURRENT, reader) : factory.create(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = factory == null ? new WhitespaceTokenizer(Version.LUCENE_CURRENT) : factory.create();
TokenStream stream = ignoreCase ? new LowerCaseFilter(Version.LUCENE_CURRENT, tokenizer) : tokenizer;
return new TokenStreamComponents(tokenizer, stream);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (working copy)
@@ -101,9 +101,8 @@
* {@link StopFilter}
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new ThaiWordFilter(matchVersion, result);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java (working copy)
@@ -119,9 +119,8 @@
* exclusion set is provided and {@link SnowballFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new TurkishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/AbstractAnalysisFactory.java (working copy)
@@ -75,7 +75,7 @@
return originalArgs;
}
- /** this method can be called in the {@link org.apache.lucene.analysis.util.TokenizerFactory#create(java.io.Reader)}
+ /** this method can be called in the {@link org.apache.lucene.analysis.util.TokenizerFactory#create()}
* or {@link org.apache.lucene.analysis.util.TokenFilterFactory#create(org.apache.lucene.analysis.TokenStream)} methods,
* to inform user, that for this factory a {@link #luceneMatchVersion} is required */
protected final void assureMatchVersion() {
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (working copy)
@@ -38,11 +38,8 @@
*
* @param matchVersion
* Lucene version to match
- * @param input
- * the input to split up into tokens
*/
- public CharTokenizer(Version matchVersion, Reader input) {
- super(input);
+ public CharTokenizer(Version matchVersion) {
charUtils = CharacterUtils.getInstance(matchVersion);
}
@@ -53,12 +50,9 @@
* Lucene version to match
* @param factory
* the attribute factory to use for this {@link Tokenizer}
- * @param input
- * the input to split up into tokens
*/
- public CharTokenizer(Version matchVersion, AttributeFactory factory,
- Reader input) {
- super(factory, input);
+ public CharTokenizer(Version matchVersion, AttributeFactory factory) {
+ super(factory);
charUtils = CharacterUtils.getInstance(matchVersion);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (working copy)
@@ -71,10 +71,10 @@
}
/** Creates a TokenStream of the specified input using the default attribute factory. */
- public final Tokenizer create(Reader input) {
- return create(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
+ public final Tokenizer create() {
+ return create(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
}
/** Creates a TokenStream of the specified input using the given AttributeFactory */
- abstract public Tokenizer create(AttributeFactory factory, Reader input);
+ abstract public Tokenizer create(AttributeFactory factory);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (working copy)
@@ -127,22 +127,18 @@
/**
* Creates a new instance of the {@link WikipediaTokenizer}. Attaches the
* input to a newly created JFlex scanner.
- *
- * @param input The Input Reader
*/
- public WikipediaTokenizer(Reader input) {
- this(input, TOKENS_ONLY, Collections.emptySet());
+ public WikipediaTokenizer() {
+ this(TOKENS_ONLY, Collections.emptySet());
}
/**
* Creates a new instance of the {@link org.apache.lucene.analysis.wikipedia.WikipediaTokenizer}. Attaches the
* input to a the newly created JFlex scanner.
*
- * @param input The input
* @param tokenOutput One of {@link #TOKENS_ONLY}, {@link #UNTOKENIZED_ONLY}, {@link #BOTH}
*/
- public WikipediaTokenizer(Reader input, int tokenOutput, Set untokenizedTypes) {
- super(input);
+ public WikipediaTokenizer(int tokenOutput, Set untokenizedTypes) {
this.scanner = new WikipediaTokenizerImpl(this.input);
init(tokenOutput, untokenizedTypes);
}
@@ -151,11 +147,10 @@
* Creates a new instance of the {@link org.apache.lucene.analysis.wikipedia.WikipediaTokenizer}. Attaches the
* input to a the newly created JFlex scanner. Uses the given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
*
- * @param input The input
* @param tokenOutput One of {@link #TOKENS_ONLY}, {@link #UNTOKENIZED_ONLY}, {@link #BOTH}
*/
- public WikipediaTokenizer(AttributeFactory factory, Reader input, int tokenOutput, Set untokenizedTypes) {
- super(factory, input);
+ public WikipediaTokenizer(AttributeFactory factory, int tokenOutput, Set untokenizedTypes) {
+ super(factory);
this.scanner = new WikipediaTokenizerImpl(this.input);
init(tokenOutput, untokenizedTypes);
}
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java (working copy)
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import java.io.Reader;
import java.util.Collections;
import java.util.Map;
@@ -45,8 +44,8 @@
// TODO: add support for WikipediaTokenizer's advanced options.
@Override
- public WikipediaTokenizer create(AttributeFactory factory, Reader input) {
- return new WikipediaTokenizer(factory, input, WikipediaTokenizer.TOKENS_ONLY,
+ public WikipediaTokenizer create(AttributeFactory factory) {
+ return new WikipediaTokenizer(factory, WikipediaTokenizer.TOKENS_ONLY,
Collections.emptySet());
}
}
Index: lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/java/org/apache/lucene/collation/CollationKeyAnalyzer.java (working copy)
@@ -87,9 +87,8 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- KeywordTokenizer tokenizer = new KeywordTokenizer(factory, reader, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ KeywordTokenizer tokenizer = new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
return new TokenStreamComponents(tokenizer, tokenizer);
}
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicFilters.java (working copy)
@@ -34,7 +34,8 @@
*/
public void testNormalizer() throws Exception {
Reader reader = new StringReader("الذين مَلكت أيمانكم");
- Tokenizer tokenizer = tokenizerFactory("Standard").create(reader);
+ Tokenizer tokenizer = tokenizerFactory("Standard").create();
+ tokenizer.setReader(reader);
TokenStream stream = tokenFilterFactory("ArabicNormalization").create(tokenizer);
assertTokenStreamContents(stream, new String[] {"الذين", "ملكت", "ايمانكم"});
}
@@ -44,7 +45,8 @@
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("الذين مَلكت أيمانكم");
- Tokenizer tokenizer = tokenizerFactory("Standard").create(reader);
+ Tokenizer tokenizer = tokenizerFactory("Standard").create();
+ tokenizer.setReader(reader);
TokenStream stream = tokenFilterFactory("ArabicNormalization").create(tokenizer);
stream = tokenFilterFactory("ArabicStem").create(stream);
assertTokenStreamContents(stream, new String[] {"ذين", "ملكت", "ايمانكم"});
@@ -55,7 +57,8 @@
*/
public void testPersianCharFilter() throws Exception {
Reader reader = charFilterFactory("Persian").create(new StringReader("میخورد"));
- Tokenizer tokenizer = tokenizerFactory("Standard").create(reader);
+ Tokenizer tokenizer = tokenizerFactory("Standard").create();
+ tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer, new String[] { "می", "خورد" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java (working copy)
@@ -89,7 +89,8 @@
}
private void check(final String input, final String expected) throws IOException {
- MockTokenizer tokenStream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenStream.setReader(new StringReader(input));
ArabicNormalizationFilter filter = new ArabicNormalizationFilter(tokenStream);
assertTokenStreamContents(filter, new String[]{expected});
}
@@ -97,8 +98,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ArabicNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (working copy)
@@ -18,8 +18,6 @@
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -122,14 +120,14 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("ساهدهات");
- MockTokenizer tokenStream = new MockTokenizer(new StringReader("ساهدهات"), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = whitespaceMockTokenizer("ساهدهات");
ArabicStemFilter filter = new ArabicStemFilter(new SetKeywordMarkerFilter(tokenStream, set));
assertTokenStreamContents(filter, new String[]{"ساهدهات"});
}
private void check(final String input, final String expected) throws IOException {
- MockTokenizer tokenStream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = whitespaceMockTokenizer(input);
ArabicStemFilter filter = new ArabicStemFilter(tokenStream);
assertTokenStreamContents(filter, new String[]{expected});
}
@@ -137,8 +135,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ArabicStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemFilterFactory.java (working copy)
@@ -34,7 +34,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("компютри");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(reader);
TokenStream stream = tokenFilterFactory("BulgarianStem").create(tokenizer);
assertTokenStreamContents(stream, new String[] { "компютр" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/bg/TestBulgarianStemmer.java (working copy)
@@ -219,7 +219,8 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("строеве");
- MockTokenizer tokenStream = new MockTokenizer(new StringReader("строевете строеве"), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenStream.setReader(new StringReader("строевете строеве"));
BulgarianStemFilter filter = new BulgarianStemFilter(
new SetKeywordMarkerFilter(tokenStream, set));
@@ -229,8 +230,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new BulgarianStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemFilterFactory.java (working copy)
@@ -34,7 +34,8 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Brasília");
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(reader);
TokenStream stream = tokenFilterFactory("BrazilianStem").create(tokenizer);
assertTokenStreamContents(stream, new String[] { "brasil" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (working copy)
@@ -146,9 +146,10 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("Brasília");
- BrazilianStemFilter filter = new BrazilianStemFilter(
- new SetKeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
- "Brasília Brasilia")), set));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("Brasília Brasilia"));
+ BrazilianStemFilter filter = new BrazilianStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
+
assertTokenStreamContents(filter, new String[] { "brasília", "brasil" });
}
@@ -168,8 +169,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new BrazilianStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/HTMLStripCharFilterTest.java (working copy)
@@ -38,8 +38,8 @@
static private Analyzer newTestAnalyzer() {
return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestHTMLStripCharFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestHTMLStripCharFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestHTMLStripCharFilterFactory.java (working copy)
@@ -35,7 +35,7 @@
// 012345678901234567890
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip", "escapedTags", "a, Title").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 0, 5, 8, 13, 15 },
@@ -47,7 +47,7 @@
// 012345678901234567890123456789012345678901
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 3, 12, 18, 27, 32 },
@@ -59,7 +59,7 @@
// 012345678901234567890123456789012345678901
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip", "escapedTags", "U i").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 0, 12, 18, 27, 29 },
@@ -71,7 +71,7 @@
// 012345678901234567890123456789012345678901
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip", "escapedTags", ",, , ").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 3, 12, 18, 27, 32 },
@@ -83,7 +83,7 @@
// 012345678901234567890123456789012345678901
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip", "escapedTags", "").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 3, 12, 18, 27, 32 },
@@ -95,7 +95,7 @@
// 012345678901234567890123456789012345678901
final String text = "this is only a test.";
Reader cs = charFilterFactory("HTMLStrip", "escapedTags", ", B\r\n\t").create(new StringReader(text));
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "only", "a", "test." },
new int[] { 3, 12, 15, 27, 32 },
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/charfilter/TestMappingCharFilter.java (working copy)
@@ -82,67 +82,67 @@
public void testNothingChange() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "x" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"x"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to1() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "h" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"i"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to2() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "j" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"jj"}, new int[]{0}, new int[]{1}, 1);
}
public void test1to3() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "k" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"kkk"}, new int[]{0}, new int[]{1}, 1);
}
public void test2to4() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "ll" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"llll"}, new int[]{0}, new int[]{2}, 2);
}
public void test2to1() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "aa" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"a"}, new int[]{0}, new int[]{2}, 2);
}
public void test3to1() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "bbb" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"b"}, new int[]{0}, new int[]{3}, 3);
}
public void test4to2() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "cccc" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"cc"}, new int[]{0}, new int[]{4}, 4);
}
public void test5to0() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "empty" ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[0], new int[]{}, new int[]{}, 5);
}
public void testNonBMPChar() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( UnicodeUtil.newString(new int[] {0x1D122}, 0, 1) ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"fclef"}, new int[]{0}, new int[]{2}, 2);
}
public void testFullWidthChar() throws Exception {
CharFilter cs = new MappingCharFilter( normMap, new StringReader( "\uff01") );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[]{"full-width-exclamation"}, new int[]{0}, new int[]{1}, 1);
}
@@ -167,7 +167,7 @@
public void testTokenStream() throws Exception {
String testString = "h i j k ll cccc bbb aa";
CharFilter cs = new MappingCharFilter( normMap, new StringReader( testString ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[]{"i","i","jj","kkk","llll","cc","b","a"},
new int[]{0,2,4,6,8,11,16,20},
@@ -190,7 +190,7 @@
String testString = "aaaa ll h";
CharFilter cs = new MappingCharFilter( normMap,
new MappingCharFilter( normMap, new StringReader( testString ) ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts =whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[]{"a","llllllll","i"},
new int[]{0,5,8},
@@ -203,8 +203,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
@@ -229,8 +229,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
@@ -251,8 +251,8 @@
final NormalizeCharMap map = randomMap();
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKAnalyzer.java (working copy)
@@ -208,8 +208,8 @@
final NormalizeCharMap norm = builder.build();
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, new CJKBigramFilter(tokenizer));
}
@@ -252,8 +252,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenFilter filter = new FakeStandardTokenizer(tokenizer);
filter = new StopFilter(TEST_VERSION_CURRENT, filter, CharArraySet.EMPTY_SET);
filter = new CJKBigramFilter(filter);
@@ -283,8 +283,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CJKBigramFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilter.java (working copy)
@@ -28,16 +28,16 @@
public class TestCJKBigramFilter extends BaseTokenStreamTestCase {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(t, new CJKBigramFilter(t));
}
};
Analyzer unibiAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(t,
new CJKBigramFilter(t, 0xff, true));
}
@@ -66,8 +66,8 @@
public void testHanOnly() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(t, new CJKBigramFilter(t, CJKBigramFilter.HAN));
}
};
@@ -84,8 +84,8 @@
public void testAllScripts() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(t,
new CJKBigramFilter(t, 0xff, false));
}
@@ -118,8 +118,8 @@
public void testUnigramsAndBigramsHanOnly() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(t, new CJKBigramFilter(t, CJKBigramFilter.HAN, true));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKBigramFilterFactory.java (working copy)
@@ -21,6 +21,7 @@
import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -29,7 +30,8 @@
public class TestCJKBigramFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testDefaults() throws Exception {
Reader reader = new StringReader("多くの学生が試験に落ちた。");
- TokenStream stream = tokenizerFactory("standard").create(reader);
+ TokenStream stream = tokenizerFactory("standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CJKBigram").create(stream);
assertTokenStreamContents(stream,
new String[] { "多く", "くの", "の学", "学生", "生が", "が試", "試験", "験に", "に落", "落ち", "ちた" });
@@ -37,7 +39,8 @@
public void testHanOnly() throws Exception {
Reader reader = new StringReader("多くの学生が試験に落ちた。");
- TokenStream stream = tokenizerFactory("standard").create(reader);
+ TokenStream stream = tokenizerFactory("standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CJKBigram",
"hiragana", "false").create(stream);
assertTokenStreamContents(stream,
@@ -46,7 +49,8 @@
public void testHanOnlyUnigrams() throws Exception {
Reader reader = new StringReader("多くの学生が試験に落ちた。");
- TokenStream stream = tokenizerFactory("standard").create(reader);
+ TokenStream stream = tokenizerFactory("standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CJKBigram",
"hiragana", "false",
"outputUnigrams", "true").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilter.java (working copy)
@@ -18,7 +18,6 @@
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -32,8 +31,8 @@
public class TestCJKWidthFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new CJKWidthFilter(source));
}
};
@@ -69,8 +68,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CJKWidthFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cjk/TestCJKWidthFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestCJKWidthFilterFactory extends BaseTokenStreamFactoryTestCase {
public void test() throws Exception {
Reader reader = new StringReader("Ｔｅｓｔ １２３４");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("CJKWidth").create(stream);
assertTokenStreamContents(stream, new String[] { "Test", "1234" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilter.java (working copy)
@@ -31,8 +31,8 @@
public class TestSoraniNormalizationFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SoraniNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniNormalizationFilterFactory.java (working copy)
@@ -31,7 +31,7 @@
public void testNormalization() throws Exception {
Reader reader = new StringReader("پیــــاوەکان");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SoraniNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "پیاوەکان" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java (working copy)
@@ -84,8 +84,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SoraniStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -31,7 +32,8 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("پیاوەکان");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("SoraniStem").create(stream);
assertTokenStreamContents(stream, new String[] { "پیاو" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/CommonGramsFilterTest.java (working copy)
@@ -35,7 +35,8 @@
public void testReset() throws Exception {
final String input = "How the s a brown s cow d like A B thing?";
- WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
+ WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
CharTermAttribute term = cgf.addAttribute(CharTermAttribute.class);
@@ -58,7 +59,8 @@
public void testQueryReset() throws Exception {
final String input = "How the s a brown s cow d like A B thing?";
- WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
+ WhitespaceTokenizer wt = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
CommonGramsQueryFilter nsf = new CommonGramsQueryFilter(cgf);
@@ -89,8 +91,8 @@
public void testCommonGramsQueryFilter() throws Exception {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader in) {
- Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new CommonGramsQueryFilter(new CommonGramsFilter(TEST_VERSION_CURRENT,
tokenizer, commonWords)));
}
@@ -159,8 +161,8 @@
public void testCommonGramsFilter() throws Exception {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader in) {
- Tokenizer tokenizer = new MockTokenizer(in, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new CommonGramsFilter(TEST_VERSION_CURRENT,
tokenizer, commonWords));
}
@@ -248,7 +250,8 @@
*/
public void testCaseSensitive() throws Exception {
final String input = "How The s a brown s cow d like A B thing?";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
TokenFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
assertTokenStreamContents(cgf, new String[] {"How", "The", "The_s", "s",
"s_a", "a", "a_brown", "brown", "brown_s", "s", "s_cow", "cow",
@@ -260,7 +263,8 @@
*/
public void testLastWordisStopWord() throws Exception {
final String input = "dog the";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
TokenFilter nsf = new CommonGramsQueryFilter(cgf);
assertTokenStreamContents(nsf, new String[] { "dog_the" });
@@ -271,7 +275,8 @@
*/
public void testFirstWordisStopWord() throws Exception {
final String input = "the dog";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
TokenFilter nsf = new CommonGramsQueryFilter(cgf);
assertTokenStreamContents(nsf, new String[] { "the_dog" });
@@ -282,7 +287,8 @@
*/
public void testOneWordQueryStopWord() throws Exception {
final String input = "the";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
TokenFilter nsf = new CommonGramsQueryFilter(cgf);
assertTokenStreamContents(nsf, new String[] { "the" });
@@ -293,7 +299,8 @@
*/
public void testOneWordQuery() throws Exception {
final String input = "monster";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
TokenFilter nsf = new CommonGramsQueryFilter(cgf);
assertTokenStreamContents(nsf, new String[] { "monster" });
@@ -304,7 +311,8 @@
*/
public void TestFirstAndLastStopWord() throws Exception {
final String input = "the of";
- MockTokenizer wt = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer wt = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ wt.setReader(new StringReader(input));
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, wt, commonWords);
TokenFilter nsf = new CommonGramsQueryFilter(cgf);
assertTokenStreamContents(nsf, new String[] { "the_of" });
@@ -315,8 +323,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, t, commonWords);
return new TokenStreamComponents(t, cgf);
}
@@ -327,8 +335,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
CommonGramsFilter cgf = new CommonGramsFilter(TEST_VERSION_CURRENT, t, commonWords);
return new TokenStreamComponents(t, new CommonGramsQueryFilter(cgf));
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsFilterFactory.java (working copy)
@@ -82,7 +82,8 @@
CharArraySet words = factory.getCommonWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue(words.contains("the"));
- Tokenizer tokenizer = new MockTokenizer(new StringReader("testing the factory"), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader("testing the factory"));
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "testing", "testing_the", "the", "the_factory", "factory" });
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsQueryFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsQueryFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/commongrams/TestCommonGramsQueryFilterFactory.java (working copy)
@@ -82,7 +82,7 @@
CharArraySet words = factory.getCommonWords();
assertTrue("words is null and it shouldn't be", words != null);
assertTrue(words.contains("the"));
- Tokenizer tokenizer = new MockTokenizer(new StringReader("testing the factory"), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer("testing the factory");
TokenStream stream = factory.create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "testing_the", "the_factory" });
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java (working copy)
@@ -53,7 +53,7 @@
.getHyphenationTree(is);
HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("min veninde som er lidt af en læsehest"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("min veninde som er lidt af en læsehest"),
hyphenator,
dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
@@ -73,7 +73,7 @@
// the word basket will not be added due to the longest match option
HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("basketballkurv"),
hyphenator, dict,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE, 40, true);
@@ -95,7 +95,7 @@
HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(
TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("basketballkurv"),
hyphenator,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
2, 4);
@@ -107,7 +107,7 @@
tf = new HyphenationCompoundWordTokenFilter(
TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("basketballkurv"),
hyphenator,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
4, 6);
@@ -119,7 +119,7 @@
tf = new HyphenationCompoundWordTokenFilter(
TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("basketballkurv"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("basketballkurv"),
hyphenator,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
4, 10);
@@ -137,11 +137,9 @@
"Pelar", "Glas", "Ögon", "Fodral", "Bas", "Fiol", "Makare", "Gesäll",
"Sko", "Vind", "Rute", "Torkare", "Blad");
- DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(
- new StringReader(
+ DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+ whitespaceMockTokenizer(
"Bildörr Bilmotor Biltak Slagborr Hammarborr Pelarborr Glasögonfodral Basfiolsfodral Basfiolsfodralmakaregesäll Skomakare Vindrutetorkare Vindrutetorkarblad abba"),
- MockTokenizer.WHITESPACE, false),
dict);
assertTokenStreamContents(tf, new String[] { "Bildörr", "Bil", "dörr", "Bilmotor",
@@ -168,7 +166,7 @@
"Sko", "Vind", "Rute", "Torkare", "Blad", "Fiolsfodral");
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new MockTokenizer(new StringReader("Basfiolsfodralmakaregesäll"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Basfiolsfodralmakaregesäll"),
dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE, true);
@@ -182,11 +180,10 @@
public void testTokenEndingWithWordComponentOfMinimumLength() throws Exception {
CharArraySet dict = makeDictionary("ab", "cd", "ef");
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("abcdef"));
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader(
- "abcdef")
- ),
+ tokenizer,
dict,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
@@ -203,11 +200,10 @@
public void testWordComponentWithLessThanMinimumLength() throws Exception {
CharArraySet dict = makeDictionary("abc", "d", "efg");
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("abcdefg"));
DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
- new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader(
- "abcdefg")
- ),
+ tokenizer,
dict,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
@@ -226,9 +222,9 @@
CharArraySet dict = makeDictionary("Rind", "Fleisch", "Draht", "Schere", "Gesetz",
"Aufgabe", "Überwachung");
- Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(
- "Rindfleischüberwachungsgesetz"));
- DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+ Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ wsTokenizer.setReader(new StringReader("Rindfleischüberwachungsgesetz"));
+ DictionaryCompoundWordTokenFilter tf = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT,
wsTokenizer, dict,
CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,
CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,
@@ -250,8 +246,8 @@
public void testRetainMockAttribute() throws Exception {
CharArraySet dict = makeDictionary("abc", "d", "efg");
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader("abcdefg"));
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("abcdefg"));
TokenStream stream = new MockRetainAttributeFilter(tokenizer);
stream = new DictionaryCompoundWordTokenFilter(
TEST_VERSION_CURRENT, stream, dict,
@@ -324,8 +320,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenFilter filter = new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, dict);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -348,8 +344,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, dict));
}
};
@@ -360,8 +356,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenFilter filter = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, hyphenator);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -374,8 +370,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new DictionaryCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, dict));
}
};
@@ -386,8 +382,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
TokenFilter filter = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT, tokenizer, hyphenator);
return new TokenStreamComponents(tokenizer, filter);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestDictionaryCompoundWordTokenFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestDictionaryCompoundWordTokenFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestDictionaryCompoundWordTokenFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testDecompounding() throws Exception {
Reader reader = new StringReader("I like to play softball");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("DictionaryCompoundWord",
"dictionary", "compoundDictionary.txt").create(stream);
assertTokenStreamContents(stream,
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testHyphenationWithDictionary() throws Exception {
Reader reader = new StringReader("min veninde som er lidt af en læsehest");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("HyphenationCompoundWord",
"hyphenator", "da_UTF8.xml",
"dictionary", "da_compoundDictionary.txt").create(stream);
@@ -51,7 +53,8 @@
*/
public void testHyphenationOnly() throws Exception {
Reader reader = new StringReader("basketballkurv");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("HyphenationCompoundWord",
"hyphenator", "da_UTF8.xml",
"minSubwordSize", "2",
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAllAnalyzersHaveFactories.java (working copy)
@@ -131,7 +131,7 @@
if (instance instanceof ResourceLoaderAware) {
((ResourceLoaderAware) instance).inform(loader);
}
- assertSame(c, instance.create(new StringReader("")).getClass());
+ assertSame(c, instance.create().getClass());
} catch (IllegalArgumentException e) {
if (!e.getMessage().contains("SPI")) {
throw e;
@@ -149,7 +149,7 @@
if (instance instanceof ResourceLoaderAware) {
((ResourceLoaderAware) instance).inform(loader);
}
- Class createdClazz = instance.create(new KeywordTokenizer(new StringReader(""))).getClass();
+ Class createdClazz = instance.create(new KeywordTokenizer()).getClass();
// only check instance if factory have wrapped at all!
if (KeywordTokenizer.class != createdClazz) {
assertSame(c, createdClazz);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestAnalyzers.java (working copy)
@@ -94,11 +94,13 @@
public void testPayloadCopy() throws IOException {
String s = "how now brown cow";
TokenStream ts;
- ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
+ ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ ((Tokenizer)ts).setReader(new StringReader(s));
ts = new PayloadSetter(ts);
verifyPayload(ts);
- ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
+ ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ ((Tokenizer)ts).setReader(new StringReader(s));
ts = new PayloadSetter(ts);
verifyPayload(ts);
}
@@ -121,8 +123,8 @@
private static class LowerCaseWhitespaceAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, new LowerCaseFilter(TEST_VERSION_CURRENT, tokenizer));
}
@@ -131,8 +133,8 @@
private static class UpperCaseWhitespaceAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, new UpperCaseFilter(TEST_VERSION_CURRENT, tokenizer));
}
@@ -188,8 +190,8 @@
public void testLowerCaseFilterLowSurrogateLeftover() throws IOException {
// test if the limit of the termbuffer is correctly used with supplementary
// chars
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader("BogustermBogusterm\udc16"));
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("BogustermBogusterm\udc16"));
LowerCaseFilter filter = new LowerCaseFilter(TEST_VERSION_CURRENT,
tokenizer);
assertTokenStreamContents(filter, new String[] {"bogustermbogusterm\udc16"});
@@ -206,16 +208,16 @@
public void testLowerCaseTokenizer() throws IOException {
StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
- LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT,
- reader);
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer, new String[] { "tokenizer",
"\ud801\udc44test" });
}
public void testWhitespaceTokenizer() throws IOException {
StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- reader);
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
"\ud801\udc1ctest" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestBugInSomething.java (working copy)
@@ -56,8 +56,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(reader), MockTokenFilter.ENGLISH_STOPSET, false, -65);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(MockTokenFilter.ENGLISH_STOPSET, false, -65);
TokenFilter f = new CommonGramsFilter(TEST_VERSION_CURRENT, t, cas);
return new TokenStreamComponents(t, f);
}
@@ -66,6 +66,7 @@
protected Reader initReader(String fieldName, Reader reader) {
reader = new MockCharFilter(reader, 0);
reader = new MappingCharFilter(map, reader);
+ reader = new TestRandomChains.CheckThatYouDidntReadAnythingReaderWrapper(reader);
return reader;
}
};
@@ -244,8 +245,8 @@
public void testUnicodeShinglesAndNgrams() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, 2, 94);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 2, 94);
//TokenStream stream = new SopTokenFilter(tokenizer);
TokenStream stream = new ShingleFilter(tokenizer, 5);
//stream = new SopTokenFilter(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestDuelingAnalyzers.java (working copy)
@@ -70,8 +70,8 @@
Analyzer left = new MockAnalyzer(random, jvmLetter, false);
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -90,8 +90,8 @@
left.setMaxTokenLength(255); // match CharTokenizer's max token length
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -108,8 +108,8 @@
Analyzer left = new MockAnalyzer(random, jvmLetter, false);
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -127,8 +127,8 @@
left.setMaxTokenLength(255); // match CharTokenizer's max token length
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -145,8 +145,8 @@
Analyzer left = new MockAnalyzer(random(), jvmLetter, false);
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -164,8 +164,8 @@
left.setMaxTokenLength(255); // match CharTokenizer's max token length
Analyzer right = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (working copy)
@@ -160,8 +160,8 @@
// some silly classes just so we can use checkRandomData
private TokenizerFactory assertingTokenizer = new TokenizerFactory(new HashMap()) {
@Override
- public MockTokenizer create(AttributeFactory factory, Reader input) {
- return new MockTokenizer(factory, input);
+ public MockTokenizer create(AttributeFactory factory) {
+ return new MockTokenizer(factory);
}
};
@@ -178,8 +178,8 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tf = tokenizer.create(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tf = tokenizer.create();
if (tokenfilter != null) {
return new TokenStreamComponents(tf, tokenfilter.create(tf));
} else {
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java (working copy)
@@ -578,13 +578,11 @@
return (T) producer.create(random);
}
- static Object[] newTokenizerArgs(Random random, Reader reader, Class[] paramTypes) {
+ static Object[] newTokenizerArgs(Random random, Class[] paramTypes) {
Object[] args = new Object[paramTypes.length];
for (int i = 0; i < args.length; i++) {
Class paramType = paramTypes[i];
- if (paramType == Reader.class) {
- args[i] = reader;
- } else if (paramType == AttributeFactory.class) {
+ if (paramType == AttributeFactory.class) {
// TODO: maybe the collator one...???
args[i] = AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY;
} else if (paramType == AttributeSource.class) {
@@ -637,15 +635,15 @@
public boolean offsetsAreCorrect() {
// TODO: can we not do the full chain here!?
Random random = new Random(seed);
- TokenizerSpec tokenizerSpec = newTokenizer(random, new StringReader(""));
+ TokenizerSpec tokenizerSpec = newTokenizer(random);
TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
return filterSpec.offsetsAreCorrect;
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
Random random = new Random(seed);
- TokenizerSpec tokenizerSpec = newTokenizer(random, reader);
+ TokenizerSpec tokenizerSpec = newTokenizer(random);
//System.out.println("seed=" + seed + ",create tokenizer=" + tokenizerSpec.toString);
TokenFilterSpec filterSpec = newFilterChain(random, tokenizerSpec.tokenizer, tokenizerSpec.offsetsAreCorrect);
//System.out.println("seed=" + seed + ",create filter=" + filterSpec.toString);
@@ -668,7 +666,7 @@
sb.append(charFilterSpec.toString);
// intentional: initReader gets its own separate random
random = new Random(seed);
- TokenizerSpec tokenizerSpec = newTokenizer(random, charFilterSpec.reader);
+ TokenizerSpec tokenizerSpec = newTokenizer(random);
sb.append("\n");
sb.append("tokenizer=");
sb.append(tokenizerSpec.toString);
@@ -726,13 +724,12 @@
}
// create a new random tokenizer from classpath
- private TokenizerSpec newTokenizer(Random random, Reader reader) {
+ private TokenizerSpec newTokenizer(Random random) {
TokenizerSpec spec = new TokenizerSpec();
while (spec.tokenizer == null) {
final Constructor ctor = tokenizers.get(random.nextInt(tokenizers.size()));
final StringBuilder descr = new StringBuilder();
- final CheckThatYouDidntReadAnythingReaderWrapper wrapper = new CheckThatYouDidntReadAnythingReaderWrapper(reader);
- final Object args[] = newTokenizerArgs(random, wrapper, ctor.getParameterTypes());
+ final Object args[] = newTokenizerArgs(random, ctor.getParameterTypes());
if (broken(ctor, args)) {
continue;
}
@@ -740,8 +737,6 @@
if (spec.tokenizer != null) {
spec.offsetsAreCorrect &= !brokenOffsets(ctor, args);
spec.toString = descr.toString();
- } else {
- assertFalse(ctor.getDeclaringClass().getName() + " has read something in ctor but failed with UOE/IAE", wrapper.readSomething);
}
}
return spec;
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java (working copy)
@@ -41,16 +41,16 @@
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- StandardTokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
+ StandardTokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(input));
BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents
- (String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer);
}
};
@@ -250,8 +250,8 @@
checkRandomData(random,
new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream tokenStream = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, tokenStream);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStopFilter.java (working copy)
@@ -38,7 +38,9 @@
public void testExactCase() throws IOException {
StringReader reader = new StringReader("Now is The Time");
CharArraySet stopWords = new CharArraySet(TEST_VERSION_CURRENT, asSet("is", "the", "Time"), false);
- TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopWords);
+ final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in.setReader(reader);
+ TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, in, stopWords);
assertTokenStreamContents(stream, new String[] { "Now", "The" });
}
@@ -46,7 +48,9 @@
StringReader reader = new StringReader("Now is The Time");
String[] stopWords = new String[] { "is", "the", "Time" };
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
- TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
+ final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in.setReader(reader);
+ TokenStream stream = new StopFilter(TEST_VERSION_CURRENT, in, stopSet);
assertTokenStreamContents(stream, new String[] { "Now", "The" });
}
@@ -67,7 +71,9 @@
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords);
// with increments
StringReader reader = new StringReader(sb.toString());
- StopFilter stpf = new StopFilter(Version.LUCENE_40, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet);
+ final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in.setReader(reader);
+ StopFilter stpf = new StopFilter(Version.LUCENE_40, in, stopSet);
doTestStopPositons(stpf);
// with increments, concatenating two stop filters
ArrayList a0 = new ArrayList();
@@ -86,7 +92,9 @@
CharArraySet stopSet0 = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords0);
CharArraySet stopSet1 = StopFilter.makeStopSet(TEST_VERSION_CURRENT, stopWords1);
reader = new StringReader(sb.toString());
- StopFilter stpf0 = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false), stopSet0); // first part of the set
+ final MockTokenizer in1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in1.setReader(reader);
+ StopFilter stpf0 = new StopFilter(TEST_VERSION_CURRENT, in1, stopSet0); // first part of the set
StopFilter stpf01 = new StopFilter(TEST_VERSION_CURRENT, stpf0, stopSet1); // two stop filters concatenated!
doTestStopPositons(stpf01);
}
@@ -94,7 +102,9 @@
// LUCENE-3849: make sure after .end() we see the "ending" posInc
public void testEndStopword() throws Exception {
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "of");
- StopFilter stpf = new StopFilter(TEST_VERSION_CURRENT, new MockTokenizer(new StringReader("test of"), MockTokenizer.WHITESPACE, false), stopSet);
+ final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in.setReader(new StringReader("test of"));
+ StopFilter stpf = new StopFilter(TEST_VERSION_CURRENT, in, stopSet);
assertTokenStreamContents(stpf, new String[] { "test" },
new int[] {0},
new int[] {4},
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (working copy)
@@ -36,7 +36,9 @@
public void testTypeFilter() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set stopTypes = asSet("");
- TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes);
+ final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT);
+ input.setReader(reader);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, input, stopTypes);
assertTokenStreamContents(stream, new String[]{"is", "palindrome", "while", "is", "not"});
}
@@ -59,7 +61,9 @@
// with increments
StringReader reader = new StringReader(sb.toString());
- TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopSet);
+ final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT);
+ input.setReader(reader);
+ TypeTokenFilter typeTokenFilter = new TypeTokenFilter(TEST_VERSION_CURRENT, input, stopSet);
testPositons(typeTokenFilter);
}
@@ -81,7 +85,9 @@
public void testTypeFilterWhitelist() throws IOException {
StringReader reader = new StringReader("121 is palindrome, while 123 is not");
Set stopTypes = Collections.singleton("");
- TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true);
+ final StandardTokenizer input = new StandardTokenizer(TEST_VERSION_CURRENT);
+ input.setReader(reader);
+ TokenStream stream = new TypeTokenFilter(TEST_VERSION_CURRENT, input, stopTypes, true);
assertTokenStreamContents(stream, new String[]{"121", "123"});
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java (working copy)
@@ -46,16 +46,16 @@
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, new StringReader(input));
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(input));
BaseTokenStreamTestCase.assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents
- (String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
- Tokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, reader);
+ Tokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
return new TokenStreamComponents(tokenizer);
}
};
@@ -101,8 +101,8 @@
private Analyzer urlAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
tokenizer.setMaxTokenLength(Integer.MAX_VALUE); // Tokenize arbitrary length URLs
TokenFilter filter = new URLFilter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
@@ -111,8 +111,8 @@
private Analyzer emailAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(TEST_VERSION_CURRENT);
TokenFilter filter = new EmailFilter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("angličtí");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CzechStem").create(stream);
assertTokenStreamContents(stream, new String[] { "anglick" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/cz/TestCzechStemmer.java (working copy)
@@ -281,16 +281,18 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("hole");
+ final MockTokenizer in = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ in.setReader(new StringReader("hole desek"));
CzechStemFilter filter = new CzechStemFilter(new SetKeywordMarkerFilter(
- new MockTokenizer(new StringReader("hole desek"), MockTokenizer.WHITESPACE, false), set));
+ in, set));
assertTokenStreamContents(filter, new String[] { "hole", "desk" });
}
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CzechStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanAnalyzer.java (working copy)
@@ -37,9 +37,10 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("fischen");
+ final LowerCaseTokenizer in = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ in.setReader(new StringReader("Fischen Trinken"));
GermanStemFilter filter = new GermanStemFilter(
- new SetKeywordMarkerFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(
- "Fischen Trinken")), set));
+ new SetKeywordMarkerFilter(in, set));
assertTokenStreamContents(filter, new String[] { "fischen", "trink" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestGermanLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new GermanLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanLightStemFilter(sink));
}
@@ -70,8 +69,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GermanLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestGermanLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("häuser");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GermanLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "haus" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new GermanMinimalStemFilter(source));
}
};
@@ -60,8 +59,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanMinimalStemFilter(sink));
}
@@ -82,8 +81,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GermanMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestGermanMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bilder");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("GermanMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "bild" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilter.java (working copy)
@@ -33,8 +33,8 @@
public class TestGermanNormalizationFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String field, Reader reader) {
- final Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String field) {
+ final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream stream = new GermanNormalizationFilter(tokenizer);
return new TokenStreamComponents(tokenizer, stream);
}
@@ -70,8 +70,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GermanNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanNormalizationFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestGermanNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("weißbier");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("GermanNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "weissbier" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilter.java (working copy)
@@ -19,7 +19,6 @@
import java.io.IOException;
import java.io.InputStream;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -42,9 +41,8 @@
public class TestGermanStemFilter extends BaseTokenStreamTestCase {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer t = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new KeywordTokenizer();
return new TokenStreamComponents(t,
new GermanStemFilter(new LowerCaseFilter(TEST_VERSION_CURRENT, t)));
}
@@ -66,8 +64,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sängerinnen"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GermanStemFilter(sink));
}
@@ -83,8 +81,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GermanStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("Tischen");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("GermanStem").create(stream);
assertTokenStreamContents(stream, new String[] { "tisch" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekLowerCaseFilterFactory.java (working copy)
@@ -33,7 +33,7 @@
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("Μάϊος ΜΆΪΟΣ");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GreekLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "μαιοσ", "μαιοσ" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestGreekStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("άνθρωπος");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GreekLowerCase").create(stream);
stream = tokenFilterFactory("GreekStem").create(stream);
assertTokenStreamContents(stream, new String[] { "ανθρωπ" });
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/el/TestGreekStemmer.java (working copy)
@@ -531,8 +531,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GreekStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilter.java (working copy)
@@ -32,9 +32,8 @@
public class TestEnglishMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new EnglishMinimalStemFilter(source));
}
};
@@ -60,8 +59,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new EnglishMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestEnglishMinimalStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestEnglishMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("EnglishMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestKStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("bricks");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KStem").create(stream);
assertTokenStreamContents(stream, new String[] { "brick" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java (working copy)
@@ -34,8 +34,8 @@
public class TestKStemmer extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer));
}
};
@@ -57,8 +57,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new KStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java (working copy)
@@ -38,9 +38,8 @@
public class TestPorterStemFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer t = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer( MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(t, new PorterStemFilter(t));
}
};
@@ -56,7 +55,8 @@
public void testWithKeywordAttribute() throws IOException {
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, 1, true);
set.add("yourselves");
- Tokenizer tokenizer = new MockTokenizer(new StringReader("yourselves yours"), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader("yourselves yours"));
TokenStream filter = new PorterStemFilter(new SetKeywordMarkerFilter(tokenizer, set));
assertTokenStreamContents(filter, new String[] {"yourselves", "your"});
}
@@ -69,8 +69,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PorterStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilterFactory.java (working copy)
@@ -33,7 +33,7 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dogs");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PorterStem").create(stream);
assertTokenStreamContents(stream, new String[] { "dog" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java (working copy)
@@ -34,9 +34,8 @@
public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new SpanishLightStemFilter(source));
}
};
@@ -54,8 +53,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SpanishLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestSpanishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("sociedades");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("SpanishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "sociedad" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianCharFilter.java (working copy)
@@ -26,8 +26,8 @@
public class TestPersianCharFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer());
}
@Override
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilter.java (working copy)
@@ -58,7 +58,7 @@
}
private void check(final String input, final String expected) throws IOException {
- MockTokenizer tokenStream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenStream = whitespaceMockTokenizer(input);
PersianNormalizationFilter filter = new PersianNormalizationFilter(
tokenStream);
assertTokenStreamContents(filter, new String[]{expected});
@@ -67,8 +67,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PersianNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fa/TestPersianNormalizationFilterFactory.java (working copy)
@@ -33,7 +33,7 @@
*/
public void testNormalization() throws Exception {
Reader reader = new StringReader("های");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PersianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "هاي" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java (working copy)
@@ -18,7 +18,6 @@
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -37,9 +36,8 @@
public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FinnishLightStemFilter(source));
}
};
@@ -53,8 +51,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("edeltäjistään"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FinnishLightStemFilter(sink));
}
@@ -70,8 +68,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FinnishLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestFinnishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("aseistettujen");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("FinnishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "aseistet" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchLightStemFilter(source));
}
};
@@ -183,8 +182,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchLightStemFilter(sink));
}
@@ -200,8 +199,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestFrenchLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("administrativement");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "administratif" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(source));
}
};
@@ -62,8 +61,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("chevaux"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer( MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new FrenchMinimalStemFilter(sink));
}
@@ -84,8 +83,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new FrenchMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestFrenchMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("chevaux");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("FrenchMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cheval" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilter.java (working copy)
@@ -18,12 +18,8 @@
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -37,8 +33,7 @@
* Test lowercase
*/
public void testIrishLowerCaseFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "nAthair tUISCE hARD"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("nAthair tUISCE hARD");
IrishLowerCaseFilter filter = new IrishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"n-athair", "t-uisce",
"hard",});
@@ -47,8 +42,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IrishLowerCaseFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ga/TestIrishLowerCaseFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestIrishLowerCaseFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testCasing() throws Exception {
Reader reader = new StringReader("nAthair tUISCE hARD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("IrishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "n-athair", "t-uisce", "hard" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilter.java (working copy)
@@ -35,8 +35,8 @@
public class TestGalicianMinimalStemFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
@@ -57,8 +57,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("elefantes"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new GalicianMinimalStemFilter(sink));
}
@@ -74,8 +74,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianMinimalStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestGalicianMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("elefantes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "elefante" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java (working copy)
@@ -36,9 +36,8 @@
public class TestGalicianStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new GalicianStemFilter(result));
}
@@ -53,8 +52,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new GalicianStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestGalicianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("cariñosa");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("GalicianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "cariñ" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiFilters.java (working copy)
@@ -21,7 +21,9 @@
import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.util.TokenizerFactory;
/**
* Simple tests to ensure the Hindi filter Factories are working.
@@ -32,7 +34,8 @@
*/
public void testIndicNormalizer() throws Exception {
Reader reader = new StringReader("ত্ अाैर");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ৎ", "और" });
}
@@ -42,7 +45,8 @@
*/
public void testHindiNormalizer() throws Exception {
Reader reader = new StringReader("क़िताब");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
assertTokenStreamContents(stream, new String[] {"किताब"});
@@ -53,7 +57,8 @@
*/
public void testStemmer() throws Exception {
Reader reader = new StringReader("किताबें");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
+ TokenStream stream = tokenizerFactory("Standard").create();
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndicNormalization").create(stream);
stream = tokenFilterFactory("HindiNormalization").create(stream);
stream = tokenFilterFactory("HindiStem").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiNormalizer.java (working copy)
@@ -62,7 +62,7 @@
check("आईऊॠॡऐऔीूॄॣैौ", "अइउऋऌएओिुृॢेो");
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -70,8 +70,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hi/TestHindiStemmer.java (working copy)
@@ -18,12 +18,9 @@
*/
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
@@ -84,7 +81,7 @@
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
TokenFilter tf = new HindiStemFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -92,8 +89,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HindiStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new HungarianLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("babakocsi"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new HungarianLightStemFilter(sink));
}
@@ -65,8 +64,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HungarianLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestHungarianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("házakat");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HungarianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "haz" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/HunspellStemFilterTest.java (working copy)
@@ -18,8 +18,6 @@
import java.io.IOException;
import java.io.InputStream;
-import java.io.Reader;
-import java.io.StringReader;
import java.text.ParseException;
import java.util.Arrays;
@@ -56,13 +54,13 @@
* Simple test for KeywordAttribute
*/
public void testKeywordAttribute() throws IOException {
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ MockTokenizer tokenizer = whitespaceMockTokenizer("lucene is awesome");
tokenizer.setEnableChecks(true);
HunspellStemFilter filter = new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "lucen", "is", "awesome"}, new int[] {1, 0, 1, 1});
// assert with keywork marker
- tokenizer = new MockTokenizer(new StringReader("lucene is awesome"), MockTokenizer.WHITESPACE, true);
+ tokenizer = whitespaceMockTokenizer("lucene is awesome");
CharArraySet set = new CharArraySet(TEST_VERSION_CURRENT, Arrays.asList("Lucene"), true);
filter = new HunspellStemFilter(new SetKeywordMarkerFilter(tokenizer, set), DICTIONARY, _TestUtil.nextInt(random(), 1, 3));
assertTokenStreamContents(filter, new String[]{"lucene", "is", "awesome"}, new int[] {1, 1, 1});
@@ -73,8 +71,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
@@ -84,8 +82,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HunspellStemFilter(tokenizer, DICTIONARY, _TestUtil.nextInt(random(), 1, 3)));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspellStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestHunspellStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("abc");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("HunspellStem",
"dictionary", "test.dic",
"affix", "test.aff").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testStemming() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "buku" });
}
@@ -43,7 +45,8 @@
*/
public void testStemmingInflectional() throws Exception {
Reader reader = new StringReader("dibukukannya");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("IndonesianStem", "stemDerivational", "false").create(stream);
assertTokenStreamContents(stream, new String[] { "dibukukan" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/id/TestIndonesianStemmer.java (working copy)
@@ -32,8 +32,8 @@
/* full stemming, no stopwords */
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
@@ -113,8 +113,8 @@
/* inflectional-only stemming */
Analyzer b = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer, false));
}
};
@@ -136,8 +136,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndonesianStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicNormalizer.java (working copy)
@@ -47,7 +47,8 @@
}
private void check(String input, String output) throws IOException {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);;
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);;
+ tokenizer.setReader(new StringReader(input));
TokenFilter tf = new IndicNormalizationFilter(tokenizer);
assertTokenStreamContents(tf, new String[] { output });
}
@@ -55,8 +56,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new IndicNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java (working copy)
@@ -34,9 +34,8 @@
public class TestItalianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new ItalianLightStemFilter(source));
}
};
@@ -54,8 +53,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ItalianLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestItalianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("ragazzo ragazzi");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ItalianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "ragazz", "ragazz" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestLatvianStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("tirgiem tirgus");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("LatvianStem").create(stream);
assertTokenStreamContents(stream, new String[] { "tirg", "tirg" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/lv/TestLatvianStemmer.java (working copy)
@@ -32,8 +32,8 @@
public class TestLatvianStemmer extends BaseTokenStreamTestCase {
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
@@ -273,8 +273,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new LatvianStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestASCIIFoldingFilter.java (working copy)
@@ -26,8 +26,6 @@
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.List;
import java.util.ArrayList;
import java.util.Iterator;
@@ -36,10 +34,9 @@
// testLain1Accents() is a copy of TestLatin1AccentFilter.testU().
public void testLatin1Accents() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader
- ("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ"
+ TokenStream stream = whitespaceMockTokenizer("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ"
+" Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij"
- +" ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl"), MockTokenizer.WHITESPACE, false);
+ +" ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl");
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
@@ -1897,7 +1894,7 @@
expectedOutputTokens.add(expected.toString());
}
- TokenStream stream = new MockTokenizer(new StringReader(inputText.toString()), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(inputText.toString());
ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream);
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
Iterator expectedIter = expectedOutputTokens.iterator();
@@ -1918,8 +1915,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
@@ -1929,8 +1926,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ASCIIFoldingFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilter.java (working copy)
@@ -107,8 +107,10 @@
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
- expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
+ expected, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength,
maxWordCount, maxTokenLength);
}
@@ -116,7 +118,9 @@
boolean onlyFirstWord, CharArraySet keep, boolean forceFirstLetter,
Collection okPrefix, int minWordLength, int maxWordCount,
int maxTokenLength) throws IOException {
- assertCapitalizesTo(new MockTokenizer(new StringReader(input), MockTokenizer.KEYWORD, false),
+ final MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ tokenizer.setReader(new StringReader(input));
+ assertCapitalizesTo(tokenizer,
new String[] { expected }, onlyFirstWord, keep, forceFirstLetter, okPrefix,
minWordLength, maxWordCount, maxTokenLength);
}
@@ -126,8 +130,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
@@ -138,8 +142,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CapitalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCapitalizationFilterFactory.java (working copy)
@@ -28,7 +28,7 @@
public void testCapitalization() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true").create(stream);
@@ -37,7 +37,7 @@
public void testCapitalization2() throws Exception {
Reader reader = new StringReader("and");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -48,7 +48,7 @@
/** first is forced, but it's not a keep word, either */
public void testCapitalization3() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -58,7 +58,7 @@
public void testCapitalization4() throws Exception {
Reader reader = new StringReader("AnD");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -68,7 +68,7 @@
public void testCapitalization5() throws Exception {
Reader reader = new StringReader("big");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -78,7 +78,7 @@
public void testCapitalization6() throws Exception {
Reader reader = new StringReader("BIG");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -88,7 +88,7 @@
public void testCapitalization7() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "true",
@@ -98,7 +98,7 @@
public void testCapitalization8() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -108,7 +108,7 @@
public void testCapitalization9() throws Exception {
Reader reader = new StringReader("Hello thEre my Name is Ryan");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -119,7 +119,7 @@
public void testCapitalization10() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -131,7 +131,7 @@
/** using "McK" as okPrefix */
public void testCapitalization11() throws Exception {
Reader reader = new StringReader("McKinley");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -144,7 +144,7 @@
/** test with numbers */
public void testCapitalization12() throws Exception {
Reader reader = new StringReader("1st 2nd third");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -156,7 +156,7 @@
public void testCapitalization13() throws Exception {
Reader reader = new StringReader("the The the");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "and the it BIG",
"onlyFirstWord", "false",
@@ -168,7 +168,7 @@
public void testKeepIgnoreCase() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -180,7 +180,7 @@
public void testKeepIgnoreCase2() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"keepIgnoreCase", "true",
@@ -192,7 +192,7 @@
public void testKeepIgnoreCase3() throws Exception {
Reader reader = new StringReader("kiTTEN");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keepIgnoreCase", "true",
"onlyFirstWord", "true",
@@ -208,7 +208,7 @@
*/
public void testMinWordLength() throws Exception {
Reader reader = new StringReader("helo testing");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"onlyFirstWord", "true",
"minWordLength", "5").create(stream);
@@ -221,7 +221,7 @@
*/
public void testMaxWordCount() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "One", "Two", "Three", "Four" });
@@ -232,7 +232,7 @@
*/
public void testMaxWordCount2() throws Exception {
Reader reader = new StringReader("one two three four");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxWordCount", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "one two three four" });
@@ -245,7 +245,7 @@
*/
public void testMaxTokenLength() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"maxTokenLength", "2").create(stream);
assertTokenStreamContents(stream, new String[] { "this", "is", "A", "test" });
@@ -256,7 +256,7 @@
*/
public void testForceFirstLetterWithKeep() throws Exception {
Reader reader = new StringReader("kitten");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Capitalization",
"keep", "kitten",
"forceFirstLetter", "true").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilter.java (working copy)
@@ -31,8 +31,7 @@
public class TestCodepointCountFilter extends BaseTokenStreamTestCase {
public void testFilterWithPosIncr() throws Exception {
- TokenStream stream = new MockTokenizer(
- new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("short toolong evenmuchlongertext a ab toolong foo");
CodepointCountFilter filter = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
@@ -43,8 +42,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new CodepointCountFilter(TEST_VERSION_CURRENT, tokenizer, 0, 5));
}
};
@@ -58,7 +57,8 @@
int max = _TestUtil.nextInt(random(), 0, 100);
int count = text.codePointCount(0, text.length());
boolean expected = count >= min && count <= max;
- TokenStream stream = new KeywordTokenizer(new StringReader(text));
+ TokenStream stream = new KeywordTokenizer();
+ ((Tokenizer)stream).setReader(new StringReader(text));
stream = new CodepointCountFilter(TEST_VERSION_CURRENT, stream, min, max);
stream.reset();
assertEquals(expected, stream.incrementToken());
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestCodepointCountFilterFactory.java (working copy)
@@ -22,13 +22,15 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
public class TestCodepointCountFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testPositionIncrements() throws Exception {
Reader reader = new StringReader("foo foobar super-duper-trooper");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("CodepointCount",
"min", "4",
"max", "10").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestHyphenatedWordsFilter.java (working copy)
@@ -35,7 +35,8 @@
public void testHyphenatedWords() throws Exception {
String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecologi-\ncal";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecological" });
@@ -45,17 +46,19 @@
* Test that HyphenatedWordsFilter behaves correctly with a final hyphen
*/
public void testHyphenAtEnd() throws Exception {
- String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
+ String input = "ecologi-\r\ncal devel-\r\n\r\nop compre-\u0009hensive-hands-on and ecology-";
// first test
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
- ts = new HyphenatedWordsFilter(ts);
- assertTokenStreamContents(ts,
- new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
+ ts = new HyphenatedWordsFilter(ts);
+ assertTokenStreamContents(ts,
+ new String[] { "ecological", "develop", "comprehensive-hands-on", "and", "ecology-" });
}
public void testOffsets() throws Exception {
String input = "abc- def geh 1234- 5678-";
- TokenStream ts = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)ts).setReader(new StringReader(input));
ts = new HyphenatedWordsFilter(ts);
assertTokenStreamContents(ts,
new String[] { "abcdef", "geh", "12345678-" },
@@ -68,8 +71,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};
@@ -80,8 +83,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new HyphenatedWordsFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeepWordFilter.java (working copy)
@@ -17,8 +17,6 @@
package org.apache.lucene.analysis.miscellaneous;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
@@ -41,12 +39,12 @@
String input = "xxx yyy aaa zzz BBB ccc ddd EEE";
// Test Stopwords
- TokenStream stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(input);
stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT, words, true));
assertTokenStreamContents(stream, new String[] { "aaa", "BBB" }, new int[] { 3, 2 });
// Now force case
- stream = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ stream = whitespaceMockTokenizer(input);
stream = new KeepWordFilter(TEST_VERSION_CURRENT, stream, new CharArraySet(TEST_VERSION_CURRENT,words, false));
assertTokenStreamContents(stream, new String[] { "aaa" }, new int[] { 3 });
}
@@ -60,8 +58,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream stream = new KeepWordFilter(TEST_VERSION_CURRENT, tokenizer, new CharArraySet(TEST_VERSION_CURRENT, words, true));
return new TokenStreamComponents(tokenizer, stream);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilter.java (working copy)
@@ -43,16 +43,13 @@
String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), set)), output);
CharArraySet mixedCaseSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("LuceneFox"), false);
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), mixedCaseSet)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), mixedCaseSet)), output);
CharArraySet set2 = set;
assertTokenStreamContents(new LowerCaseFilterMock(
- new SetKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), set2)), output);
+ new SetKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), set2)), output);
}
@Test
@@ -60,15 +57,13 @@
String[] output = new String[] { "the", "quick", "brown", "LuceneFox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[fF]ox"))), output);
+ new PatternKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), Pattern.compile("[a-zA-Z]+[fF]ox"))), output);
output = new String[] { "the", "quick", "brown", "lucenefox",
"jumps" };
assertTokenStreamContents(new LowerCaseFilterMock(
- new PatternKeywordMarkerFilter(new MockTokenizer(new StringReader(
- "The quIck browN LuceneFox Jumps"), MockTokenizer.WHITESPACE, false), Pattern.compile("[a-zA-Z]+[f]ox"))), output);
+ new PatternKeywordMarkerFilter(whitespaceMockTokenizer("The quIck browN LuceneFox Jumps"), Pattern.compile("[a-zA-Z]+[f]ox"))), output);
}
// LUCENE-2901
@@ -76,7 +71,7 @@
TokenStream ts = new LowerCaseFilterMock(
new SetKeywordMarkerFilter(
new SetKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Birds", "Houses"), false)),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));
@@ -85,7 +80,7 @@
ts = new LowerCaseFilterMock(
new PatternKeywordMarkerFilter(
new PatternKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
Pattern.compile("Birds|Houses")),
Pattern.compile("Dogs|Trees")));
@@ -94,7 +89,7 @@
ts = new LowerCaseFilterMock(
new SetKeywordMarkerFilter(
new PatternKeywordMarkerFilter(
- new MockTokenizer(new StringReader("Dogs Trees Birds Houses"), MockTokenizer.WHITESPACE, false),
+ whitespaceMockTokenizer("Dogs Trees Birds Houses"),
Pattern.compile("Birds|Houses")),
new CharArraySet(TEST_VERSION_CURRENT, asSet("Dogs", "Trees"), false)));
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordMarkerFilterFactory.java (working copy)
@@ -20,7 +20,6 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
@@ -32,7 +31,7 @@
public void testKeywords() throws Exception {
Reader reader = new StringReader("dogs cats");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker", TEST_VERSION_CURRENT,
new StringMockResourceLoader("cats"),
"protected", "protwords.txt").create(stream);
@@ -42,7 +41,7 @@
public void testKeywords2() throws Exception {
Reader reader = new StringReader("dogs cats");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker",
"pattern", "cats|Dogs").create(stream);
stream = tokenFilterFactory("PorterStem").create(stream);
@@ -51,7 +50,7 @@
public void testKeywordsMixed() throws Exception {
Reader reader = new StringReader("dogs cats birds");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker", TEST_VERSION_CURRENT,
new StringMockResourceLoader("cats"),
"protected", "protwords.txt",
@@ -62,7 +61,7 @@
public void testKeywordsCaseInsensitive() throws Exception {
Reader reader = new StringReader("dogs cats Cats");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker", TEST_VERSION_CURRENT,
new StringMockResourceLoader("cats"),
"protected", "protwords.txt",
@@ -73,7 +72,7 @@
public void testKeywordsCaseInsensitive2() throws Exception {
Reader reader = new StringReader("dogs cats Cats");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker",
"pattern", "Cats",
"ignoreCase", "true").create(stream);
@@ -83,7 +82,7 @@
public void testKeywordsCaseInsensitiveMixed() throws Exception {
Reader reader = new StringReader("dogs cats Cats Birds birds");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("KeywordMarker", TEST_VERSION_CURRENT,
new StringMockResourceLoader("cats"),
"protected", "protwords.txt",
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordRepeatFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordRepeatFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestKeywordRepeatFilter.java (working copy)
@@ -18,25 +18,23 @@
*/
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import java.io.IOException;
-import java.io.StringReader;
public class TestKeywordRepeatFilter extends BaseTokenStreamTestCase {
public void testBasic() throws IOException {
TokenStream ts = new RemoveDuplicatesTokenFilter(new SnowballFilter(new KeywordRepeatFilter(
- new MockTokenizer(new StringReader("the birds are flying"), MockTokenizer.WHITESPACE, false)), "English"));
+ whitespaceMockTokenizer("the birds are flying")), "English"));
assertTokenStreamContents(ts, new String[] { "the", "birds", "bird", "are", "flying", "fli"}, new int[] {1,1,0,1,1,0});
}
public void testComposition() throws IOException {
TokenStream ts = new RemoveDuplicatesTokenFilter(new SnowballFilter(new KeywordRepeatFilter(new KeywordRepeatFilter(
- new MockTokenizer(new StringReader("the birds are flying"), MockTokenizer.WHITESPACE, false))), "English"));
+ whitespaceMockTokenizer("the birds are flying"))), "English"));
assertTokenStreamContents(ts, new String[] { "the", "birds", "bird", "are", "flying", "fli"}, new int[] {1,1,0,1,1,0});
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java (working copy)
@@ -31,8 +31,7 @@
public class TestLengthFilter extends BaseTokenStreamTestCase {
public void testFilterWithPosIncr() throws Exception {
- TokenStream stream = new MockTokenizer(
- new StringReader("short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("short toolong evenmuchlongertext a ab toolong foo");
LengthFilter filter = new LengthFilter(TEST_VERSION_CURRENT, stream, 2, 6);
assertTokenStreamContents(filter,
new String[]{"short", "ab", "foo"},
@@ -43,8 +42,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new LengthFilter(TEST_VERSION_CURRENT, tokenizer, 0, 5));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilterFactory.java (working copy)
@@ -21,13 +21,15 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
public class TestLengthFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testPositionIncrements() throws Exception {
Reader reader = new StringReader("foo foobar super-duper-trooper");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Length",
"min", "4",
"max", "10").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountFilterFactory.java (working copy)
@@ -27,7 +27,8 @@
public void test() throws Exception {
Reader reader = new StringReader("A1 B2 C3 D4 E5 F6");
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(reader);
// LimitTokenCountFilter doesn't consume the entire stream that it wraps
tokenizer.setEnableChecks(false);
TokenStream stream = tokenizer;
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilter.java (working copy)
@@ -34,8 +34,8 @@
for (final boolean consumeAll : new boolean[] { true, false }) {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
// if we are consuming all tokens, we can use the checks, otherwise we can't
tokenizer.setEnableChecks(consumeAll);
return new TokenStreamComponents(tokenizer, new LimitTokenPositionFilter(tokenizer, 2, consumeAll));
@@ -59,7 +59,7 @@
}
public void testMaxPosition3WithSynomyms() throws IOException {
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("one two three four five"), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = whitespaceMockTokenizer("one two three four five");
tokenizer.setEnableChecks(false); // LimitTokenPositionFilter doesn't consume the entire stream that it wraps
SynonymMap.Builder builder = new SynonymMap.Builder(true);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenPositionFilterFactory.java (working copy)
@@ -27,7 +27,7 @@
public void testMaxPosition1() throws Exception {
Reader reader = new StringReader("A1 B2 C3 D4 E5 F6");
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = whitespaceMockTokenizer(reader);
// LimitTokenPositionFilter doesn't consume the entire stream that it wraps
tokenizer.setEnableChecks(false);
TokenStream stream = tokenizer;
@@ -48,7 +48,7 @@
public void testMaxPosition1WithShingles() throws Exception {
Reader reader = new StringReader("one two three four five");
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = whitespaceMockTokenizer(reader);
// LimitTokenPositionFilter doesn't consume the entire stream that it wraps
tokenizer.setEnableChecks(false);
TokenStream stream = tokenizer;
@@ -63,7 +63,7 @@
public void testConsumeAllTokens() throws Exception {
Reader reader = new StringReader("A1 B2 C3 D4 E5 F6");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("LimitTokenPosition",
"maxTokenPosition", "3",
"consumeAllTokens", "true").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalyzerWrapper.java (working copy)
@@ -65,8 +65,8 @@
public void testCharFilters() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer());
}
@Override
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java (working copy)
@@ -28,9 +28,11 @@
public void test() throws IOException {
+ final MockTokenizer input = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ input.setReader(new StringReader("hello world"));
PrefixAndSuffixAwareTokenFilter ts = new PrefixAndSuffixAwareTokenFilter(
new SingleTokenTokenStream(createToken("^", 0, 0)),
- new MockTokenizer(new StringReader("hello world"), MockTokenizer.WHITESPACE, false),
+ input,
new SingleTokenTokenStream(createToken("$", 0, 0)));
assertTokenStreamContents(ts,
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java (working copy)
@@ -40,8 +40,10 @@
// prefix and suffix using 2x prefix
+ final MockTokenizer suffix = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ suffix.setReader(new StringReader("hello world"));
ts = new PrefixAwareTokenFilter(new SingleTokenTokenStream(createToken("^", 0, 0)),
- new MockTokenizer(new StringReader("hello world"), MockTokenizer.WHITESPACE, false));
+ suffix);
ts = new PrefixAwareTokenFilter(ts, new SingleTokenTokenStream(createToken("$", 0, 0)));
assertTokenStreamContents(ts,
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestRemoveDuplicatesTokenFilter.java (working copy)
@@ -156,8 +156,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
TokenStream stream = new SynonymFilter(tokenizer, map, ignoreCase);
return new TokenStreamComponents(tokenizer, new RemoveDuplicatesTokenFilter(stream));
}
@@ -170,8 +170,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new RemoveDuplicatesTokenFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilter.java (working copy)
@@ -31,8 +31,8 @@
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String field, Reader reader) {
- final Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String field) {
+ final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream stream = new ScandinavianFoldingFilter(tokenizer);
return new TokenStreamComponents(tokenizer, stream);
}
@@ -111,8 +111,8 @@
public void testEmptyTerm() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ScandinavianFoldingFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianFoldingFilterFactory.java (working copy)
@@ -27,7 +27,7 @@
public void testStemming() throws Exception {
Reader reader = new StringReader("räksmörgås");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ScandinavianFolding").create(stream);
assertTokenStreamContents(stream, new String[] { "raksmorgas" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilter.java (working copy)
@@ -32,8 +32,8 @@
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String field, Reader reader) {
- final Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String field) {
+ final Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream stream = new ScandinavianNormalizationFilter(tokenizer);
return new TokenStreamComponents(tokenizer, stream);
}
@@ -110,8 +110,8 @@
public void testEmptyTerm() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ScandinavianNormalizationFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestScandinavianNormalizationFilterFactory.java (working copy)
@@ -26,8 +26,7 @@
public class TestScandinavianNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
- Reader reader = new StringReader("räksmörgås");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("räksmörgås");
stream = tokenFilterFactory("ScandinavianNormalization").create(stream);
assertTokenStreamContents(stream, new String[] { "ræksmørgås" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilter.java (working copy)
@@ -37,13 +37,21 @@
*
*/
public class TestStemmerOverrideFilter extends BaseTokenStreamTestCase {
+
+ private KeywordTokenizer keywordTokenizer(String data) throws IOException {
+ KeywordTokenizer tokenizer = new KeywordTokenizer();
+ tokenizer.setReader(new StringReader(data));
+ return tokenizer;
+ }
+
+
public void testOverride() throws IOException {
// lets make booked stem to books
// the override filter will convert "booked" to "books",
// but also mark it with KeywordAttribute so Porter will not change it.
StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder();
builder.add("booked", "books");
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader("booked"));
+ Tokenizer tokenizer = keywordTokenizer("booked");
TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(
tokenizer, builder.build()));
assertTokenStreamContents(stream, new String[] {"books"});
@@ -55,7 +63,7 @@
// but also mark it with KeywordAttribute so Porter will not change it.
StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(true);
builder.add("boOkEd", "books");
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader("BooKeD"));
+ Tokenizer tokenizer = keywordTokenizer("BooKeD");
TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(
tokenizer, builder.build()));
assertTokenStreamContents(stream, new String[] {"books"});
@@ -63,7 +71,7 @@
public void testNoOverrides() throws IOException {
StemmerOverrideFilter.Builder builder = new StemmerOverrideFilter.Builder(true);
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader("book"));
+ Tokenizer tokenizer = keywordTokenizer("book");
TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(
tokenizer, builder.build()));
assertTokenStreamContents(stream, new String[] {"book"});
@@ -105,8 +113,8 @@
output.add(entry.getValue());
}
}
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader(input.toString()));
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(input.toString()));
TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(
tokenizer, builder.build()));
assertTokenStreamContents(stream, output.toArray(new String[0]));
@@ -135,8 +143,8 @@
StemmerOverrideMap build = builder.build();
for (Entry entry : entrySet) {
if (random().nextBoolean()) {
- Tokenizer tokenizer = new KeywordTokenizer(new StringReader(
- entry.getKey()));
+ Tokenizer tokenizer = new KeywordTokenizer();
+ tokenizer.setReader(new StringReader(entry.getKey()));
TokenStream stream = new PorterStemFilter(new StemmerOverrideFilter(
tokenizer, build));
assertTokenStreamContents(stream, new String[] {entry.getValue()});
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestStemmerOverrideFilterFactory.java (working copy)
@@ -20,7 +20,6 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
@@ -32,7 +31,7 @@
public void testKeywords() throws Exception {
// our stemdict stems dogs to 'cat'
Reader reader = new StringReader("testing dogs");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("StemmerOverride", TEST_VERSION_CURRENT,
new StringMockResourceLoader("dogs\tcat"),
"dictionary", "stemdict.txt").create(stream);
@@ -43,7 +42,7 @@
public void testKeywordsCaseInsensitive() throws Exception {
Reader reader = new StringReader("testing DoGs");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("StemmerOverride", TEST_VERSION_CURRENT,
new StringMockResourceLoader("dogs\tcat"),
"dictionary", "stemdict.txt",
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilter.java (working copy)
@@ -98,8 +98,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new TrimFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
@@ -109,8 +109,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
final Version version = TEST_VERSION_CURRENT;
return new TokenStreamComponents(tokenizer, new TrimFilter(version, tokenizer));
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestTrimFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestTrimFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testTrimming() throws Exception {
Reader reader = new StringReader("trim me ");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ TokenStream stream = keywordMockTokenizer(reader);
stream = tokenFilterFactory("Trim").create(stream);
assertTokenStreamContents(stream, new String[] { "trim me" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestWordDelimiterFilter.java (working copy)
@@ -18,10 +18,8 @@
package org.apache.lucene.analysis.miscellaneous;
import org.apache.lucene.analysis.*;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.cz.CzechStemFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -29,8 +27,6 @@
import org.junit.Test;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.*;
import static org.apache.lucene.analysis.miscellaneous.WordDelimiterFilter.*;
@@ -130,8 +126,8 @@
public void doSplit(final String input, String... output) throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS | STEM_ENGLISH_POSSESSIVE;
- WordDelimiterFilter wdf = new WordDelimiterFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.KEYWORD, false), WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, flags, null);
+ WordDelimiterFilter wdf = new WordDelimiterFilter(keywordMockTokenizer(input),
+ WordDelimiterIterator.DEFAULT_WORD_DELIM_TABLE, flags, null);
assertTokenStreamContents(wdf, output);
}
@@ -174,8 +170,7 @@
public void doSplitPossessive(int stemPossessive, final String input, final String... output) throws Exception {
int flags = GENERATE_WORD_PARTS | GENERATE_NUMBER_PARTS | SPLIT_ON_CASE_CHANGE | SPLIT_ON_NUMERICS;
flags |= (stemPossessive == 1) ? STEM_ENGLISH_POSSESSIVE : 0;
- WordDelimiterFilter wdf = new WordDelimiterFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.KEYWORD, false), flags, null);
+ WordDelimiterFilter wdf = new WordDelimiterFilter(keywordMockTokenizer(input), flags, null);
assertTokenStreamContents(wdf, output);
}
@@ -220,8 +215,8 @@
/* analyzer that uses whitespace + wdf */
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(
tokenizer,
flags, protWords));
@@ -257,8 +252,8 @@
/* analyzer that will consume tokens with large position increments */
Analyzer a2 = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(
new LargePosIncTokenFilter(tokenizer),
flags, protWords));
@@ -302,8 +297,8 @@
Analyzer a3 = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
StopFilter filter = new StopFilter(TEST_VERSION_CURRENT,
tokenizer, StandardAnalyzer.STOP_WORDS_SET);
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(filter, flags, protWords));
@@ -345,8 +340,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(tokenizer, flags, protectedWords));
}
};
@@ -367,8 +362,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new WordDelimiterFilter(tokenizer, flags, protectedWords));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilterTest.java (working copy)
@@ -46,7 +46,7 @@
@Override
public void setUp() throws Exception {
super.setUp();
- input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
+ input = whitespaceMockTokenizer("abcde");
}
public void testInvalidInput() throws Exception {
@@ -95,7 +95,7 @@
}
public void testFilterPositions() throws Exception {
- TokenStream ts = new MockTokenizer(new StringReader("abcde vwxyz"), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer("abcde vwxyz");
EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, ts, 1, 3);
assertTokenStreamContents(tokenizer,
new String[]{"a","ab","abc","v","vw","vwx"},
@@ -139,7 +139,7 @@
}
public void testFirstTokenPositionIncrement() throws Exception {
- TokenStream ts = new MockTokenizer(new StringReader("a abc"), MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer("a abc");
ts = new PositionFilter(ts); // All but first token will get 0 position increment
EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, ts, 2, 3);
// The first token "a" will not be output, since it's smaller than the mingram size of 2.
@@ -154,13 +154,14 @@
}
public void testSmallTokenInStream() throws Exception {
- input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
+ input = whitespaceMockTokenizer("abc de fgh");
EdgeNGramTokenFilter tokenizer = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, input, 3, 3);
assertTokenStreamContents(tokenizer, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10});
}
public void testReset() throws Exception {
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("abcde"));
EdgeNGramTokenFilter filter = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 1, 3);
assertTokenStreamContents(filter, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{5,5,5});
tokenizer.setReader(new StringReader("abcde"));
@@ -175,8 +176,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, min, max));
}
@@ -189,8 +190,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer,
new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 2, 15));
}
@@ -199,7 +200,8 @@
}
public void testGraphs() throws IOException {
- TokenStream tk = new LetterTokenizer(TEST_VERSION_CURRENT, new StringReader("abc d efgh ij klmno p q"));
+ TokenStream tk = new LetterTokenizer(TEST_VERSION_CURRENT);
+ ((Tokenizer)tk).setReader(new StringReader("abc d efgh ij klmno p q"));
tk = new ShingleFilter(tk);
tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, 7, 10);
assertTokenStreamContents(tk,
@@ -217,7 +219,8 @@
final int codePointCount = s.codePointCount(0, s.length());
final int minGram = _TestUtil.nextInt(random(), 1, 3);
final int maxGram = _TestUtil.nextInt(random(), minGram, 10);
- TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ TokenStream tk = new KeywordTokenizer();
+ ((Tokenizer)tk).setReader(new StringReader(s));
tk = new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerTest.java (working copy)
@@ -45,7 +45,7 @@
public void testInvalidInput() throws Exception {
boolean gotException = false;
try {
- new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 0, 0);
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 0, 0).setReader(input);
} catch (IllegalArgumentException e) {
gotException = true;
}
@@ -55,7 +55,7 @@
public void testInvalidInput2() throws Exception {
boolean gotException = false;
try {
- new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 2, 1);
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 2, 1).setReader(input);
} catch (IllegalArgumentException e) {
gotException = true;
}
@@ -65,7 +65,7 @@
public void testInvalidInput3() throws Exception {
boolean gotException = false;
try {
- new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, -1, 2);
+ new EdgeNGramTokenizer(TEST_VERSION_CURRENT, -1, 2).setReader(input);
} catch (IllegalArgumentException e) {
gotException = true;
}
@@ -73,22 +73,26 @@
}
public void testFrontUnigram() throws Exception {
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 1, 1);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"a"}, new int[]{0}, new int[]{1}, 5 /* abcde */);
}
public void testOversizedNgrams() throws Exception {
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 6, 6);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 6, 6);
+ tokenizer.setReader(input);;
assertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
}
public void testFrontRangeOfNgrams() throws Exception {
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 1, 3);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
}
public void testReset() throws Exception {
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 1, 3);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","ab","abc"}, new int[]{0,0,0}, new int[]{1,2,3}, 5 /* abcde */);
@@ -102,8 +106,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, min, max);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -113,7 +117,8 @@
}
public void testTokenizerPositions() throws Exception {
- EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"), 1, 3);
+ EdgeNGramTokenizer tokenizer = new EdgeNGramTokenizer(TEST_VERSION_CURRENT, 1, 3);
+ tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer,
new String[]{"a","ab","abc"},
new int[]{0,0,0},
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (working copy)
@@ -45,7 +45,7 @@
@Override
public void setUp() throws Exception {
super.setUp();
- input = new MockTokenizer(new StringReader("abcde"), MockTokenizer.WHITESPACE, false);
+ input = whitespaceMockTokenizer("abcde");
}
public void testInvalidInput() throws Exception {
@@ -108,13 +108,14 @@
}
public void testSmallTokenInStream() throws Exception {
- input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false);
+ input = whitespaceMockTokenizer("abc de fgh");
NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 3, 3);
assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10}, new int[] {1, 2});
}
public void testReset() throws Exception {
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde"));
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader("abcde"));
NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 1, 1);
assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,0,0,0,0}, new int[]{5,5,5,5,5}, new int[]{1,0,0,0,0});
tokenizer.setReader(new StringReader("abcde"));
@@ -128,8 +129,8 @@
public void testInvalidOffsets() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
filters = new NGramTokenFilter(TEST_VERSION_CURRENT, filters, 2, 2);
return new TokenStreamComponents(tokenizer, filters);
@@ -149,8 +150,8 @@
final int max = _TestUtil.nextInt(random(), min, 20);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, min, max));
}
@@ -163,8 +164,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer,
new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 2, 15));
}
@@ -189,7 +190,8 @@
final int codePointCount = s.codePointCount(0, s.length());
final int minGram = _TestUtil.nextInt(random(), 1, 3);
final int maxGram = _TestUtil.nextInt(random(), minGram, 10);
- TokenStream tk = new KeywordTokenizer(new StringReader(s));
+ TokenStream tk = new KeywordTokenizer();
+ ((Tokenizer)tk).setReader(new StringReader(s));
tk = new NGramTokenFilter(TEST_VERSION_CURRENT, tk, minGram, maxGram);
final CharTermAttribute termAtt = tk.addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = tk.addAttribute(OffsetAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (working copy)
@@ -51,8 +51,9 @@
public void testInvalidInput() throws Exception {
boolean gotException = false;
- try {
- new NGramTokenizer(TEST_VERSION_CURRENT, input, 2, 1);
+ try {
+ NGramTokenizer tok = new NGramTokenizer(TEST_VERSION_CURRENT, 2, 1);
+ tok.setReader(input);
} catch (IllegalArgumentException e) {
gotException = true;
}
@@ -61,8 +62,9 @@
public void testInvalidInput2() throws Exception {
boolean gotException = false;
- try {
- new NGramTokenizer(TEST_VERSION_CURRENT, input, 0, 1);
+ try {
+ NGramTokenizer tok = new NGramTokenizer(TEST_VERSION_CURRENT, 0, 1);
+ tok.setReader(input);
} catch (IllegalArgumentException e) {
gotException = true;
}
@@ -70,17 +72,20 @@
}
public void testUnigrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, 1, 1);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
}
public void testBigrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 2, 2);
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, 2, 2);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}, 5 /* abcde */);
}
public void testNgrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 3);
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, 1, 3);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer,
new String[]{"a","ab", "abc", "b", "bc", "bcd", "c", "cd", "cde", "d", "de", "e"},
new int[]{0,0,0,1,1,1,2,2,2,3,3,4},
@@ -94,12 +99,14 @@
}
public void testOversizedNgrams() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 6, 7);
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, 6, 7);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[0], new int[0], new int[0], 5 /* abcde */);
}
public void testReset() throws Exception {
- NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, input, 1, 1);
+ NGramTokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, 1, 1);
+ tokenizer.setReader(input);
assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
tokenizer.setReader(new StringReader("abcde"));
assertTokenStreamContents(tokenizer, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, 5 /* abcde */);
@@ -112,8 +119,8 @@
final int max = _TestUtil.nextInt(random(), min, 20);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, reader, min, max);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new NGramTokenizer(TEST_VERSION_CURRENT, min, max);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -158,12 +165,13 @@
for (int i = 0; i < codePoints.length; ++i) {
offsets[i+1] = offsets[i] + Character.charCount(codePoints[i]);
}
- final TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram, edgesOnly) {
+ final Tokenizer grams = new NGramTokenizer(TEST_VERSION_CURRENT, minGram, maxGram, edgesOnly) {
@Override
protected boolean isTokenChar(int chr) {
return nonTokenChars.indexOf(chr) < 0;
}
};
+ grams.setReader(new StringReader(s));
final CharTermAttribute termAtt = grams.addAttribute(CharTermAttribute.class);
final PositionIncrementAttribute posIncAtt = grams.addAttribute(PositionIncrementAttribute.class);
final PositionLengthAttribute posLenAtt = grams.addAttribute(PositionLengthAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/TestNGramFilters.java (working copy)
@@ -20,8 +20,8 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,9 +33,10 @@
*/
public void testNGramTokenizer() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = tokenizerFactory("NGram").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "t", "te", "e", "es", "s", "st", "t" });
+ TokenStream stream = tokenizerFactory("NGram").create();
+ ((Tokenizer)stream).setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"t", "te", "e", "es", "s", "st", "t"});
}
/**
@@ -45,7 +46,8 @@
Reader reader = new StringReader("test");
TokenStream stream = tokenizerFactory("NGram",
"minGramSize", "2",
- "maxGramSize", "3").create(reader);
+ "maxGramSize", "3").create();
+ ((Tokenizer)stream).setReader(reader);
assertTokenStreamContents(stream,
new String[] { "te", "tes", "es", "est", "st" });
}
@@ -55,7 +57,7 @@
*/
public void testNGramFilter() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("NGram").create(stream);
assertTokenStreamContents(stream,
new String[] { "t", "te", "e", "es", "s", "st", "t" });
@@ -66,7 +68,7 @@
*/
public void testNGramFilter2() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("NGram",
"minGramSize", "2",
"maxGramSize", "3").create(stream);
@@ -79,7 +81,8 @@
*/
public void testEdgeNGramTokenizer() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = tokenizerFactory("EdgeNGram").create(reader);
+ TokenStream stream = tokenizerFactory("EdgeNGram").create();
+ ((Tokenizer)stream).setReader(reader);
assertTokenStreamContents(stream,
new String[] { "t" });
}
@@ -91,7 +94,8 @@
Reader reader = new StringReader("test");
TokenStream stream = tokenizerFactory("EdgeNGram",
"minGramSize", "1",
- "maxGramSize", "2").create(reader);
+ "maxGramSize", "2").create();
+ ((Tokenizer)stream).setReader(reader);
assertTokenStreamContents(stream,
new String[] { "t", "te" });
}
@@ -101,7 +105,7 @@
*/
public void testEdgeNGramFilter() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("EdgeNGram").create(stream);
assertTokenStreamContents(stream,
new String[] { "t" });
@@ -112,7 +116,7 @@
*/
public void testEdgeNGramFilter2() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("EdgeNGram",
"minGramSize", "1",
"maxGramSize", "2").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java (working copy)
@@ -42,9 +42,8 @@
public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, BOKMAAL));
}
};
@@ -58,8 +57,8 @@
public void testNynorskVocabulary() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NYNORSK));
}
};
@@ -70,8 +69,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new NorwegianLightStemFilter(sink));
}
@@ -88,8 +87,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new NorwegianLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestNorwegianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("epler eple");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("NorwegianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl" });
}
@@ -38,7 +38,7 @@
/** Test stemming with variant set explicitly to Bokmål */
public void testBokmaalStemming() throws Exception {
Reader reader = new StringReader("epler eple");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("NorwegianLightStem", "variant", "nb").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl" });
}
@@ -46,7 +46,7 @@
/** Test stemming with variant set explicitly to Nynorsk */
public void testNynorskStemming() throws Exception {
Reader reader = new StringReader("gutar gutane");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("NorwegianLightStem", "variant", "nn").create(stream);
assertTokenStreamContents(stream, new String[] { "gut", "gut" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java (working copy)
@@ -41,9 +41,8 @@
public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, BOKMAAL));
}
};
@@ -57,8 +56,8 @@
public void testNynorskVocabulary() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, NYNORSK));
}
};
@@ -69,8 +68,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("sekretæren"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(sink));
}
@@ -87,8 +86,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new NorwegianMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestNorwegianMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("eple eplet epler eplene eplets eplenes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("NorwegianMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
}
@@ -38,7 +40,8 @@
/** Test stemming with variant set explicitly to Bokmål */
public void testBokmaalStemming() throws Exception {
Reader reader = new StringReader("eple eplet epler eplene eplets eplenes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nb").create(stream);
assertTokenStreamContents(stream, new String[] { "epl", "epl", "epl", "epl", "epl", "epl" });
}
@@ -46,7 +49,8 @@
/** Test stemming with variant set explicitly to Nynorsk */
public void testNynorskStemming() throws Exception {
Reader reader = new StringReader("gut guten gutar gutane gutens gutanes");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("NorwegianMinimalStem", "variant", "nn").create(stream);
assertTokenStreamContents(stream, new String[] { "gut", "gut", "gut", "gut", "gut", "gut" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestPathHierarchyTokenizer.java (working copy)
@@ -31,7 +31,8 @@
public void testBasic() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c"},
new int[]{0, 0, 0},
@@ -42,7 +43,8 @@
public void testEndOfDelimiter() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/a", "/a/b", "/a/b/c", "/a/b/c/"},
new int[]{0, 0, 0, 0},
@@ -53,7 +55,8 @@
public void testStartOfChar() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer();
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c"},
new int[]{0, 0, 0},
@@ -64,7 +67,8 @@
public void testStartOfCharEndOfDelimiter() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"a", "a/b", "a/b/c", "a/b/c/"},
new int[]{0, 0, 0, 0},
@@ -75,7 +79,8 @@
public void testOnlyDelimiter() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader( new StringReader(path) );
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -86,7 +91,8 @@
public void testOnlyDelimiters() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path) );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/", "//"},
new int[]{0, 0},
@@ -97,7 +103,8 @@
public void testReplace() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), '/', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( '/', '\\' );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"\\a", "\\a\\b", "\\a\\b\\c"},
new int[]{0, 0, 0},
@@ -108,7 +115,8 @@
public void testWindowsPath() throws Exception {
String path = "c:\\a\\b\\c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), '\\', '\\' );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( '\\', '\\' );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"c:", "c:\\a", "c:\\a\\b", "c:\\a\\b\\c"},
new int[]{0, 0, 0, 0},
@@ -123,7 +131,8 @@
NormalizeCharMap normMap = builder.build();
String path = "c:\\a\\b\\c";
Reader cs = new MappingCharFilter(normMap, new StringReader(path));
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( cs );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( );
+ t.setReader(cs);
assertTokenStreamContents(t,
new String[]{"c:", "c:/a", "c:/a/b", "c:/a/b/c"},
new int[]{0, 0, 0, 0},
@@ -134,7 +143,8 @@
public void testBasicSkip() throws Exception {
String path = "/a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
new int[]{2, 2},
@@ -145,7 +155,8 @@
public void testEndOfDelimiterSkip() throws Exception {
String path = "/a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
new int[]{2, 2, 2},
@@ -156,7 +167,8 @@
public void testStartOfCharSkip() throws Exception {
String path = "a/b/c";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c"},
new int[]{1, 1},
@@ -167,7 +179,8 @@
public void testStartOfCharEndOfDelimiterSkip() throws Exception {
String path = "a/b/c/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer(1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/b", "/b/c", "/b/c/"},
new int[]{1, 1, 1},
@@ -178,7 +191,8 @@
public void testOnlyDelimiterSkip() throws Exception {
String path = "/";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
new int[]{},
@@ -189,7 +203,8 @@
public void testOnlyDelimitersSkip() throws Exception {
String path = "//";
- PathHierarchyTokenizer t = new PathHierarchyTokenizer( new StringReader(path), 1 );
+ PathHierarchyTokenizer t = new PathHierarchyTokenizer( 1 );
+ t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{1},
@@ -202,8 +217,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -215,8 +230,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/path/TestReversePathHierarchyTokenizer.java (working copy)
@@ -30,7 +30,8 @@
public void testBasicReverse() throws Exception {
String path = "/a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c", "a/b/c", "b/c", "c"},
new int[]{0, 1, 3, 5},
@@ -41,7 +42,8 @@
public void testEndOfDelimiterReverse() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/b/c/", "a/b/c/", "b/c/", "c/"},
new int[]{0, 1, 3, 5},
@@ -52,7 +54,8 @@
public void testStartOfCharReverse() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c", "b/c", "c"},
new int[]{0, 2, 4},
@@ -63,7 +66,8 @@
public void testStartOfCharEndOfDelimiterReverse() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/c/", "b/c/", "c/"},
new int[]{0, 2, 4},
@@ -74,7 +78,8 @@
public void testOnlyDelimiterReverse() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -85,7 +90,8 @@
public void testOnlyDelimitersReverse() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path) );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer();
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"//", "/"},
new int[]{0, 1},
@@ -96,7 +102,9 @@
public void testEndOfDelimiterReverseSkip() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
+ new StringReader(path);
assertTokenStreamContents(t,
new String[]{"/a/b/", "a/b/", "b/"},
new int[]{0, 1, 3},
@@ -107,7 +115,8 @@
public void testStartOfCharReverseSkip() throws Exception {
String path = "a/b/c";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
new int[]{0, 2},
@@ -118,7 +127,8 @@
public void testStartOfCharEndOfDelimiterReverseSkip() throws Exception {
String path = "a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"a/b/", "b/"},
new int[]{0, 2},
@@ -129,7 +139,8 @@
public void testOnlyDelimiterReverseSkip() throws Exception {
String path = "/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{},
new int[]{},
@@ -140,7 +151,8 @@
public void testOnlyDelimitersReverseSkip() throws Exception {
String path = "//";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 1 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 1 );
+ t.setReader(new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/"},
new int[]{0},
@@ -151,7 +163,8 @@
public void testReverseSkip2() throws Exception {
String path = "/a/b/c/";
- ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( new StringReader(path), 2 );
+ ReversePathHierarchyTokenizer t = new ReversePathHierarchyTokenizer( 2 );
+ t.setReader( new StringReader(path));
assertTokenStreamContents(t,
new String[]{"/a/", "a/"},
new int[]{0, 1},
@@ -164,8 +177,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -177,8 +190,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new ReversePathHierarchyTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ReversePathHierarchyTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternCaptureGroupTokenFilter.java (working copy)
@@ -597,10 +597,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader,
- MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new PatternCaptureGroupTokenFilter(tokenizer, false,
Pattern.compile("((..)(..))")));
@@ -617,9 +615,10 @@
for (int i = 0; i < regexes.length; i++) {
patterns[i] = Pattern.compile(regexes[i]);
}
- TokenStream ts = new PatternCaptureGroupTokenFilter(new MockTokenizer(
- new StringReader(input), MockTokenizer.WHITESPACE, false),
- preserveOriginal, patterns);
+
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader( new StringReader(input));
+ TokenStream ts = new PatternCaptureGroupTokenFilter(tokenizer, preserveOriginal, patterns);
assertTokenStreamContents(ts, tokens, startOffsets, endOffsets, positions);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (working copy)
@@ -139,7 +139,7 @@
final String BLOCK = "this is test.";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1$2$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
new int[] { 0, 5, 8 },
@@ -153,7 +153,7 @@
final String BLOCK = "aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts, new String[] {});
}
@@ -164,7 +164,7 @@
final String BLOCK = "aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2#$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
new int[] { 0 },
@@ -180,7 +180,7 @@
final String BLOCK = "aa bb cc dd";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1##$2###$3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa##bb###cc", "dd" },
new int[] { 0, 9 },
@@ -195,7 +195,7 @@
final String BLOCK = " a a";
CharFilter cs = new PatternReplaceCharFilter( pattern("a"), "aa",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "aa" },
new int[] { 1, 4 },
@@ -211,7 +211,7 @@
final String BLOCK = "aa bb cc dd";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1#$2",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa#bb", "dd" },
new int[] { 0, 12 },
@@ -227,7 +227,7 @@
final String BLOCK = " aa bb cc --- aa bb aa bb cc";
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)\\s+(cc)"), "$1 $2 $3",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "bb", "cc", "---", "aa", "bb", "aa", "bb", "cc" },
new int[] { 2, 6, 9, 11, 15, 18, 21, 25, 29 },
@@ -247,7 +247,7 @@
CharFilter cs = new PatternReplaceCharFilter( pattern("(aa)\\s+(bb)"), "$1##$2",
new StringReader( BLOCK ) );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa##bb", "cc", "---", "aa##bb", "aa.", "bb", "aa##bb", "cc" },
new int[] { 2, 8, 11, 15, 21, 25, 28, 36 },
@@ -265,7 +265,7 @@
new StringReader( BLOCK ) );
cs = new PatternReplaceCharFilter( pattern("bb"), "b", cs );
cs = new PatternReplaceCharFilter( pattern("ccc"), "c", cs );
- TokenStream ts = new MockTokenizer(cs, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(cs);
assertTokenStreamContents(ts,
new String[] { "aa", "b", "-", "c", ".", "---", "b", "aa", ".", "c", "c", "b" },
new int[] { 1, 3, 6, 8, 12, 14, 18, 21, 23, 25, 29, 33 },
@@ -307,8 +307,8 @@
final String replacement = _TestUtil.randomSimpleString(random);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilterFactory.java (working copy)
@@ -37,7 +37,7 @@
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)",
"replacement", "$1$2$3").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts,
new String[] { "this", "is", "test." },
new int[] { 0, 5, 8 },
@@ -50,7 +50,7 @@
Reader reader = new StringReader("aa bb cc");
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts, new String[] {});
}
@@ -62,7 +62,7 @@
reader = charFilterFactory("PatternReplace",
"pattern", "(aa)\\s+(bb)\\s+(cc)",
"replacement", "$1#$2#$3").create(reader);
- TokenStream ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream ts = whitespaceMockTokenizer(reader);
assertTokenStreamContents(ts,
new String[] { "aa#bb#cc" },
new int[] { 0 },
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java (working copy)
@@ -25,18 +25,16 @@
import org.apache.lucene.analysis.core.KeywordTokenizer;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
import java.util.regex.Pattern;
/**
*/
public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {
-
+
public void testReplaceAll() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
"-", true);
assertTokenStreamContents(ts,
@@ -46,7 +44,7 @@
public void testReplaceFirst() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
"-", false);
assertTokenStreamContents(ts,
@@ -56,7 +54,7 @@
public void testStripFirst() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
null, false);
assertTokenStreamContents(ts,
@@ -66,7 +64,7 @@
public void testStripAll() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("a*b"),
null, true);
assertTokenStreamContents(ts,
@@ -76,7 +74,7 @@
public void testReplaceAllWithBackRef() throws Exception {
String input = "aabfooaabfooabfoob ab caaaaaaaaab";
TokenStream ts = new PatternReplaceFilter
- (new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(input),
Pattern.compile("(a*)b"),
"$1\\$", true);
assertTokenStreamContents(ts,
@@ -87,8 +85,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", false);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -97,8 +95,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream filter = new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -109,8 +107,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PatternReplaceFilter(tokenizer, Pattern.compile("a"), "b", true));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilterFactory.java (working copy)
@@ -31,7 +31,7 @@
public void testReplaceAll() throws Exception {
Reader reader = new StringReader("aabfooaabfooabfoob ab caaaaaaaaab");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PatternReplace",
"pattern", "a*b",
"replacement", "-").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizer.java (working copy)
@@ -53,7 +53,8 @@
};
for( String[] test : tests ) {
- TokenStream stream = new PatternTokenizer(new StringReader(test[2]), Pattern.compile(test[1]), Integer.parseInt(test[0]));
+ TokenStream stream = new PatternTokenizer(Pattern.compile(test[1]), Integer.parseInt(test[0]));
+ ((Tokenizer)stream).setReader(new StringReader(test[2]));
String out = tsToString( stream );
// System.out.println( test[2] + " ==> " + out );
@@ -85,7 +86,8 @@
CharFilter charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
// create PatternTokenizer
- TokenStream stream = new PatternTokenizer(charStream, Pattern.compile("[,;/\\s]+"), -1);
+ Tokenizer stream = new PatternTokenizer(Pattern.compile("[,;/\\s]+"), -1);
+ stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" },
new int[] { 0, 13, 26, 29 },
@@ -93,7 +95,8 @@
INPUT.length());
charStream = new MappingCharFilter( normMap, new StringReader( INPUT ) );
- stream = new PatternTokenizer(charStream, Pattern.compile("Günther"), 0);
+ stream = new PatternTokenizer(Pattern.compile("Günther"), 0);
+ stream.setReader(charStream);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther" },
new int[] { 0, 13 },
@@ -128,8 +131,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), -1);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), -1);
return new TokenStreamComponents(tokenizer);
}
};
@@ -137,8 +140,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new PatternTokenizer(reader, Pattern.compile("a"), 0);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new PatternTokenizer(Pattern.compile("a"), 0);
return new TokenStreamComponents(tokenizer);
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternTokenizerFactory.java (working copy)
@@ -20,7 +20,7 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/** Simple Tests to ensure this factory is working */
@@ -28,7 +28,8 @@
public void testFactory() throws Exception {
final Reader reader = new StringReader("Günther Günther is here");
// create PatternTokenizer
- TokenStream stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create(reader);
+ Tokenizer stream = tokenizerFactory("Pattern", "pattern", "[,;/\\s]+").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Günther", "Günther", "is", "here" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilterTest.java (working copy)
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -25,12 +26,12 @@
import java.io.StringReader;
-public class DelimitedPayloadTokenFilterTest extends LuceneTestCase {
+public class DelimitedPayloadTokenFilterTest extends BaseTokenStreamTestCase {
public void testPayloads() throws Exception {
String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
- (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(test),
DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
@@ -54,7 +55,7 @@
String test = "The quick|JJ red|JJ fox|NN jumped|VB over the lazy|JJ brown|JJ dogs|NN";
DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter
- (new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false),
+ (whitespaceMockTokenizer(test),
DelimitedPayloadTokenFilter.DEFAULT_DELIMITER, new IdentityEncoder());
filter.reset();
assertTermEquals("The", filter, null);
@@ -75,7 +76,7 @@
public void testFloatEncoding() throws Exception {
String test = "The quick|1.0 red|2.0 fox|3.5 jumped|0.5 over the lazy|5 brown|99.3 dogs|83.7";
- DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new FloatEncoder());
+ DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(whitespaceMockTokenizer(test), '|', new FloatEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
filter.reset();
@@ -96,7 +97,7 @@
public void testIntEncoding() throws Exception {
String test = "The quick|1 red|2 fox|3 jumped over the lazy|5 brown|99 dogs|83";
- DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false), '|', new IntegerEncoder());
+ DelimitedPayloadTokenFilter filter = new DelimitedPayloadTokenFilter(whitespaceMockTokenizer(test), '|', new IntegerEncoder());
CharTermAttribute termAtt = filter.getAttribute(CharTermAttribute.class);
PayloadAttribute payAtt = filter.getAttribute(PayloadAttribute.class);
filter.reset();
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilterTest.java (working copy)
@@ -32,7 +32,9 @@
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)), 3, "D");
+ final MockTokenizer input = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ input.setReader(new StringReader(test));
+ NumericPayloadTokenFilter nptf = new NumericPayloadTokenFilter(new WordTokenFilter(input), 3, "D");
boolean seenDogs = false;
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TestDelimitedPayloadTokenFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.payloads.PayloadHelper;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
@@ -30,7 +31,8 @@
public void testEncoder() throws Exception {
Reader reader = new StringReader("the|0.1 quick|0.1 red|0.1");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("DelimitedPayload", "encoder", "float").create(stream);
stream.reset();
@@ -48,7 +50,8 @@
public void testDelim() throws Exception {
Reader reader = new StringReader("the*0.1 quick*0.1 red*0.1");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("DelimitedPayload",
"encoder", "float",
"delimiter", "*").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilterTest.java (working copy)
@@ -23,14 +23,13 @@
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
-import java.io.StringReader;
public class TokenOffsetPayloadTokenFilterTest extends BaseTokenStreamTestCase {
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TokenOffsetPayloadTokenFilter nptf = new TokenOffsetPayloadTokenFilter(whitespaceMockTokenizer(test));
int count = 0;
PayloadAttribute payloadAtt = nptf.getAttribute(PayloadAttribute.class);
OffsetAttribute offsetAtt = nptf.getAttribute(OffsetAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilterTest.java (working copy)
@@ -32,7 +32,7 @@
public void test() throws IOException {
String test = "The quick red fox jumped over the lazy brown dogs";
- TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
+ TypeAsPayloadTokenFilter nptf = new TypeAsPayloadTokenFilter(new WordTokenFilter(whitespaceMockTokenizer(test)));
int count = 0;
CharTermAttribute termAtt = nptf.getAttribute(CharTermAttribute.class);
TypeAttribute typeAtt = nptf.getAttribute(TypeAttribute.class);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java (working copy)
@@ -40,9 +40,8 @@
public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseLightStemFilter(result));
}
@@ -101,8 +100,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseLightStemFilter(sink));
}
@@ -118,8 +117,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestPortugueseLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("evidentemente");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("PortugueseLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "evident" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java (working copy)
@@ -40,9 +40,8 @@
public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(result));
}
@@ -75,8 +74,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseMinimalStemFilter(sink));
}
@@ -92,8 +91,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseMinimalStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -30,7 +31,8 @@
public class TestPortugueseMinimalStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("questões");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("PortugueseMinimalStem").create(stream);
assertTokenStreamContents(stream, new String[] { "questão" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java (working copy)
@@ -39,9 +39,8 @@
public class TestPortugueseStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new LowerCaseFilter(TEST_VERSION_CURRENT, source);
return new TokenStreamComponents(source, new PortugueseStemFilter(result));
}
@@ -74,8 +73,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("quilométricas"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new PortugueseStemFilter(sink));
}
@@ -91,8 +90,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PortugueseStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestPortugueseStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("maluquice");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("PortugueseStem").create(stream);
assertTokenStreamContents(stream, new String[] { "maluc" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilter.java (working copy)
@@ -31,15 +31,15 @@
public class TestReverseStringFilter extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader("Do have a nice day"),
- MockTokenizer.WHITESPACE, false); // 1-4 length string
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false); // 1-4 length string
+ ((Tokenizer)stream).setReader(new StringReader("Do have a nice day"));
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream);
assertTokenStreamContents(filter, new String[] { "oD", "evah", "a", "ecin", "yad" });
}
public void testFilterWithMark() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader("Do have a nice day"),
- MockTokenizer.WHITESPACE, false); // 1-4 length string
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false); // 1-4 length string
+ ((Tokenizer)stream).setReader(new StringReader("Do have a nice day"));
ReverseStringFilter filter = new ReverseStringFilter(TEST_VERSION_CURRENT, stream, '\u0001');
assertTokenStreamContents(filter,
new String[] { "\u0001oD", "\u0001evah", "\u0001a", "\u0001ecin", "\u0001yad" });
@@ -97,8 +97,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
@@ -108,8 +108,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/reverse/TestReverseStringFilterFactory.java (working copy)
@@ -33,7 +33,7 @@
*/
public void testReversing() throws Exception {
Reader reader = new StringReader("simple test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ReverseString").create(stream);
assertTokenStreamContents(stream, new String[] { "elpmis", "tset" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestRussianLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new RussianLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("энергии"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new RussianLightStemFilter(sink));
}
@@ -70,8 +69,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new RussianLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilterFactory.java (working copy)
@@ -30,7 +30,7 @@
public class TestRussianLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("журналы");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("RussianLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "журнал" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (working copy)
@@ -981,7 +981,8 @@
}
public void testReset() throws Exception {
- Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence"));
+ Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ wsTokenizer.setReader(new StringReader("please divide this sentence"));
TokenStream filter = new ShingleFilter(wsTokenizer, 2);
assertTokenStreamContents(filter,
new String[]{"please","please divide","divide","divide this","this","this sentence","sentence"},
@@ -1105,8 +1106,8 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
@@ -1118,8 +1119,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
@@ -1129,8 +1130,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ShingleFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/shingle/TestShingleFilterFactory.java (working copy)
@@ -33,7 +33,7 @@
*/
public void testDefaults() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle").create(stream);
assertTokenStreamContents(stream,
new String[] { "this", "this is", "is", "is a", "a", "a test", "test" }
@@ -45,7 +45,7 @@
*/
public void testNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"outputUnigrams", "false").create(stream);
assertTokenStreamContents(stream,
@@ -57,7 +57,7 @@
*/
public void testMaxShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"maxShingleSize", "3").create(stream);
assertTokenStreamContents(stream,
@@ -71,7 +71,7 @@
*/
public void testMinShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4").create(stream);
@@ -86,7 +86,7 @@
*/
public void testMinShingleSizeNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -100,7 +100,7 @@
*/
public void testEqualMinAndMaxShingleSize() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "3").create(stream);
@@ -113,7 +113,7 @@
*/
public void testEqualMinAndMaxShingleSizeNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "3",
@@ -127,7 +127,7 @@
*/
public void testTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "=BLAH=").create(stream);
assertTokenStreamContents(stream,
@@ -141,7 +141,7 @@
*/
public void testTokenSeparatorNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "=BLAH=",
"outputUnigrams", "false").create(stream);
@@ -154,7 +154,7 @@
*/
public void testEmptyTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"tokenSeparator", "").create(stream);
assertTokenStreamContents(stream,
@@ -167,7 +167,7 @@
*/
public void testMinShingleSizeAndTokenSeparator() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -186,7 +186,7 @@
*/
public void testMinShingleSizeAndTokenSeparatorNoUnigrams() throws Exception {
Reader reader = new StringReader("this is a test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"minShingleSize", "3",
"maxShingleSize", "4",
@@ -208,7 +208,7 @@
*/
public void testOutputUnigramsIfNoShingles() throws Exception {
Reader reader = new StringReader("test");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Shingle",
"outputUnigrams", "false",
"outputUnigramsIfNoShingles", "true").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/DateRecognizerSinkTokenizerTest.java (working copy)
@@ -29,7 +29,9 @@
public void test() throws IOException {
DateRecognizerSinkFilter sinkFilter = new DateRecognizerSinkFilter(new SimpleDateFormat("MM/dd/yyyy", Locale.ROOT));
String test = "The quick red fox jumped over the lazy brown dogs on 7/11/2006 The dogs finally reacted on 7/12/2006";
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer input = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ input.setReader(new StringReader(test));
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(input);
TeeSinkTokenFilter.SinkTokenStream sink = tee.newSinkTokenStream(sinkFilter);
int count = 0;
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -126,7 +127,7 @@
}
public void testGeneral() throws IOException {
- final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter source = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer1.toString()));
final TokenStream sink1 = source.newSinkTokenStream();
final TokenStream sink2 = source.newSinkTokenStream(theFilter);
@@ -140,7 +141,7 @@
}
public void testMultipleSources() throws Exception {
- final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer1.toString()));
final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
tee1.reset();
@@ -150,7 +151,7 @@
dogDetector.addAttribute(CheckClearAttributesAttribute.class);
theDetector.addAttribute(CheckClearAttributesAttribute.class);
- final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer2.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(whitespaceMockTokenizer(buffer2.toString()));
tee2.addSinkTokenStream(dogDetector);
tee2.addSinkTokenStream(theDetector);
final TokenStream source2 = tee2;
@@ -168,6 +169,12 @@
lowerCaseTokens[i] = tokens1[i].toLowerCase(Locale.ROOT);
assertTokenStreamContents(lowerCasing, lowerCaseTokens);
}
+
+ private StandardTokenizer standardTokenizer(StringBuilder builder) throws IOException {
+ StandardTokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
+ return tokenizer;
+ }
/**
* Not an explicit test, just useful to print out some info on performance
@@ -182,10 +189,10 @@
buffer.append(English.intToEnglish(i).toUpperCase(Locale.ROOT)).append(' ');
}
//make sure we produce the same tokens
- TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)));
TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(100));
teeStream.consumeAllTokens();
- TokenStream stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), 100);
+ TokenStream stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)), 100);
CharTermAttribute tfTok = stream.addAttribute(CharTermAttribute.class);
CharTermAttribute sinkTok = sink.addAttribute(CharTermAttribute.class);
for (int i=0; stream.incrementToken(); i++) {
@@ -198,12 +205,12 @@
int tfPos = 0;
long start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
- stream = new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString())));
+ stream = new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer));
PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
tfPos += posIncrAtt.getPositionIncrement();
}
- stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), modCounts[j]);
+ stream = new ModuloTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)), modCounts[j]);
posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
while (stream.incrementToken()) {
tfPos += posIncrAtt.getPositionIncrement();
@@ -215,7 +222,7 @@
//simulate one field with one sink
start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
- teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ teeStream = new TeeSinkTokenFilter(new StandardFilter(TEST_VERSION_CURRENT, standardTokenizer(buffer)));
sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(modCounts[j]));
PositionIncrementAttribute posIncrAtt = teeStream.getAttribute(PositionIncrementAttribute.class);
while (teeStream.incrementToken()) {
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenRangeSinkTokenizerTest.java (working copy)
@@ -27,7 +27,7 @@
public void test() throws IOException {
TokenRangeSinkFilter sinkFilter = new TokenRangeSinkFilter(2, 4);
String test = "The quick red fox jumped over the lazy brown dogs";
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false));
+ TeeSinkTokenFilter tee = new TeeSinkTokenFilter(whitespaceMockTokenizer(test));
TeeSinkTokenFilter.SinkTokenStream rangeToks = tee.newSinkTokenStream(sinkFilter);
int count = 0;
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TokenTypeSinkTokenizerTest.java (working copy)
@@ -17,10 +17,8 @@
*/
import java.io.IOException;
-import java.io.StringReader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
@@ -32,7 +30,7 @@
TokenTypeSinkFilter sinkFilter = new TokenTypeSinkFilter("D");
String test = "The quick red fox jumped over the lazy brown dogs";
- TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(new MockTokenizer(new StringReader(test), MockTokenizer.WHITESPACE, false)));
+ TeeSinkTokenFilter ttf = new TeeSinkTokenFilter(new WordTokenFilter(whitespaceMockTokenizer(test)));
TeeSinkTokenFilter.SinkTokenStream sink = ttf.newSinkTokenStream(sinkFilter);
boolean seenDogs = false;
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java (working copy)
@@ -18,16 +18,14 @@
*/
import java.io.IOException;
-import java.io.Reader;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Analyzer.TokenStreamComponents;
import org.apache.lucene.analysis.core.KeywordTokenizer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
+
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -35,15 +33,15 @@
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Version;
+
public class TestSnowball extends BaseTokenStreamTestCase {
public void testEnglish() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, "English"));
}
};
@@ -109,8 +107,8 @@
for (final String lang : SNOWBALL_LANGS) {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SnowballFilter(tokenizer, lang));
}
};
@@ -127,8 +125,8 @@
public void checkRandomStrings(final String snowballLanguage) throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer();
return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballPorterFilterFactory.java (working copy)
@@ -39,7 +39,7 @@
}
Reader reader = new StringReader(text);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SnowballPorter", "language", "English").create(stream);
assertTokenStreamContents(stream, gold);
}
@@ -49,7 +49,7 @@
*/
public void testProtected() throws Exception {
Reader reader = new StringReader("ridding of some stemming");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SnowballPorter", TEST_VERSION_CURRENT,
new StringMockResourceLoader("ridding"),
"protected", "protwords.txt",
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java (working copy)
@@ -70,9 +70,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer t = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new KeywordTokenizer();
return new TokenStreamComponents(t, new SnowballFilter(t, snowballLanguage));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestStandardFactories.java (working copy)
@@ -34,9 +34,10 @@
*/
public void testStandardTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- TokenStream stream = tokenizerFactory("Standard").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "Wha\u0301t's", "this", "thing", "do" });
+ Tokenizer stream = tokenizerFactory("Standard").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"Wha\u0301t's", "this", "thing", "do"});
}
public void testStandardTokenizerMaxTokenLength() throws Exception {
@@ -48,9 +49,10 @@
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Standard",
- "maxTokenLength", "1000").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "one", "two", "three", longWord, "four", "five", "six" });
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"one", "two", "three", longWord, "four", "five", "six"});
}
/**
@@ -58,9 +60,10 @@
*/
public void testClassicTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Classic").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "What's", "this", "thing", "do" });
+ Tokenizer stream = tokenizerFactory("Classic").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"What's", "this", "thing", "do"});
}
public void testClassicTokenizerMaxTokenLength() throws Exception {
@@ -72,9 +75,10 @@
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
Tokenizer stream = tokenizerFactory("Classic",
- "maxTokenLength", "1000").create(reader);
- assertTokenStreamContents(stream,
- new String[] { "one", "two", "three", longWord, "four", "five", "six" });
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
+ assertTokenStreamContents(stream,
+ new String[]{"one", "two", "three", longWord, "four", "five", "six"});
}
/**
@@ -82,8 +86,9 @@
*/
public void testStandardFilter() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Classic").create(reader);
- stream = tokenFilterFactory("Classic").create(stream);
+ Tokenizer tokenizer = tokenizerFactory("Classic").create();
+ tokenizer.setReader(reader);
+ TokenStream stream = tokenFilterFactory("Classic").create(tokenizer);
assertTokenStreamContents(stream,
new String[] { "What", "this", "thing", "do" });
}
@@ -93,7 +98,8 @@
*/
public void testKeywordTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Keyword").create(reader);
+ Tokenizer stream = tokenizerFactory("Keyword").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What's this thing do?" });
}
@@ -103,7 +109,8 @@
*/
public void testWhitespaceTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Whitespace").create(reader);
+ Tokenizer stream = tokenizerFactory("Whitespace").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What's", "this", "thing", "do?" });
}
@@ -113,7 +120,8 @@
*/
public void testLetterTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("Letter").create(reader);
+ Tokenizer stream = tokenizerFactory("Letter").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "What", "s", "this", "thing", "do" });
}
@@ -123,7 +131,8 @@
*/
public void testLowerCaseTokenizer() throws Exception {
Reader reader = new StringReader("What's this thing do?");
- TokenStream stream = tokenizerFactory("LowerCase").create(reader);
+ Tokenizer stream = tokenizerFactory("LowerCase").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "what", "s", "this", "thing", "do" });
}
@@ -133,7 +142,7 @@
*/
public void testASCIIFolding() throws Exception {
Reader reader = new StringReader("Česká");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ASCIIFolding").create(stream);
assertTokenStreamContents(stream, new String[] { "Ceska" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java (working copy)
@@ -20,7 +20,7 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -31,14 +31,16 @@
public void testUAX29URLEmailTokenizer() throws Exception {
Reader reader = new StringReader("Wha\u0301t's this thing do?");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Wha\u0301t's", "this", "thing", "do" });
}
public void testArabic() throws Exception {
Reader reader = new StringReader("الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "الفيلم", "الوثائقي", "الأول", "عن", "ويكيبيديا", "يسمى", "الحقيقة", "بالأرقام", "قصة", "ويكيبيديا",
"بالإنجليزية", "Truth", "in", "Numbers", "The", "Wikipedia", "Story", "سيتم", "إطلاقه", "في", "2008" });
@@ -46,21 +48,24 @@
public void testChinese() throws Exception {
Reader reader = new StringReader("我是中国人。 １２３４ Ｔｅｓｔｓ ");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "我", "是", "中", "国", "人", "１２３４", "Ｔｅｓｔｓ" });
}
public void testKorean() throws Exception {
Reader reader = new StringReader("안녕하세요 한글입니다");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "안녕하세요", "한글입니다" });
}
public void testHyphen() throws Exception {
Reader reader = new StringReader("some-dashed-phrase");
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "some", "dashed", "phrase" });
}
@@ -82,7 +87,8 @@
+ " blah Sirrah woof "
+ "http://[a42:a7b6::]/qSmxSUU4z/%52qVl4\n";
Reader reader = new StringReader(textWithURLs);
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
"http://johno.jsmf.net/knowhow/ngrams/index.php?table=en-dickens-word-2gram¶graphs=50&length=200&no-ads=on",
@@ -120,7 +126,8 @@
+ "lMahAA.j/5.RqUjS745.DtkcYdi@d2-4gb-l6.ae\n"
+ "lv'p@tqk.vj5s0tgl.0dlu7su3iyiaz.dqso.494.3hb76.XN--MGBAAM7A8H\n";
Reader reader = new StringReader(textWithEmails);
- TokenStream stream = tokenizerFactory("UAX29URLEmail").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {
"some", "extra", "Words", "thrown", "in", "here",
@@ -149,8 +156,9 @@
String longWord = builder.toString();
String content = "one two three " + longWord + " four five six";
Reader reader = new StringReader(content);
- TokenStream stream = tokenizerFactory("UAX29URLEmail",
- "maxTokenLength", "1000").create(reader);
+ Tokenizer stream = tokenizerFactory("UAX29URLEmail",
+ "maxTokenLength", "1000").create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] {"one", "two", "three", longWord, "four", "five", "six" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java (working copy)
@@ -37,9 +37,8 @@
public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new SwedishLightStemFilter(source));
}
};
@@ -53,8 +52,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("jaktkarlens"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new SwedishLightStemFilter(sink));
}
@@ -70,8 +69,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SwedishLightStemFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilterFactory.java (working copy)
@@ -20,7 +20,6 @@
import java.io.Reader;
import java.io.StringReader;
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
@@ -30,7 +29,7 @@
public class TestSwedishLightStemFilterFactory extends BaseTokenStreamFactoryTestCase {
public void testStemming() throws Exception {
Reader reader = new StringReader("äpplen äpple");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("SwedishLightStem").create(stream);
assertTokenStreamContents(stream, new String[] { "äppl", "äppl" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestMultiWordSynonyms.java (working copy)
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
@@ -32,7 +31,7 @@
public void testMultiWordSynonyms() throws Exception {
Reader reader = new StringReader("a e");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Synonym", TEST_VERSION_CURRENT,
new StringMockResourceLoader("a b c,d"),
"synonyms", "synonyms.txt").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSolrSynonymParser.java (working copy)
@@ -49,8 +49,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -122,8 +122,8 @@
final SynonymMap map = parser.build();
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymFilterFactory.java (working copy)
@@ -19,15 +19,10 @@
import java.io.Reader;
import java.io.StringReader;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.pattern.PatternTokenizerFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
-import org.apache.lucene.analysis.util.ClasspathResourceLoader;
import org.apache.lucene.analysis.util.StringMockResourceLoader;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
@@ -36,7 +31,7 @@
/** checks for synonyms of "GB" in synonyms.txt */
private void checkSolrSynonyms(TokenFilterFactory factory) throws Exception {
Reader reader = new StringReader("GB");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTrue(stream instanceof SynonymFilter);
assertTokenStreamContents(stream,
@@ -47,7 +42,7 @@
/** checks for synonyms of "second" in synonyms-wordnet.txt */
private void checkWordnetSynonyms(TokenFilterFactory factory) throws Exception {
Reader reader = new StringReader("second");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTrue(stream instanceof SynonymFilter);
assertTokenStreamContents(stream,
@@ -63,7 +58,7 @@
/** if the synonyms are completely empty, test that we still analyze correctly */
public void testEmptySynonyms() throws Exception {
Reader reader = new StringReader("GB");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("Synonym", TEST_VERSION_CURRENT,
new StringMockResourceLoader(""), // empty file!
"synonyms", "synonyms.txt").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (working copy)
@@ -151,8 +151,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
@@ -176,8 +176,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
@@ -208,9 +208,8 @@
add("z", "boo", false);
add("y", "bee", true);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
- true);
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -424,9 +423,9 @@
}
}
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE,
true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -496,8 +495,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -525,7 +524,7 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, new SynonymFilter(graph, map, ignoreCase));
@@ -552,8 +551,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
TokenStream syns = new SynonymFilter(tokenizer, map, ignoreCase);
TokenStream graph = new MockGraphTokenFilter(random(), syns);
return new TokenStreamComponents(tokenizer, graph);
@@ -578,8 +577,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -607,8 +606,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, ignoreCase));
}
};
@@ -629,8 +628,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -650,9 +649,9 @@
final boolean keepOrig = false;
add("aaa", "aaaa1 aaaa2 aaaa3", keepOrig);
add("bbb", "bbbb1 bbbb2", keepOrig);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE,
true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -688,8 +687,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -728,8 +727,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -748,8 +747,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -766,8 +765,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -785,8 +784,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -802,9 +801,8 @@
final boolean keepOrig = false;
// b hangs off the end (no input token under it):
add("a", "a b", keepOrig);
- tokensIn = new MockTokenizer(new StringReader("a"),
- MockTokenizer.WHITESPACE,
- true);
+ tokensIn = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ tokensIn.setReader(new StringReader("a"));
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
@@ -835,8 +833,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -880,8 +878,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -899,8 +897,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
@@ -917,8 +915,8 @@
final SynonymMap map = b.build();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestWordnetSynonymParser.java (working copy)
@@ -46,8 +46,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, false));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (working copy)
@@ -131,8 +131,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ThaiWordFilter(TEST_VERSION_CURRENT, tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiWordFilterFactory.java (working copy)
@@ -35,7 +35,7 @@
public void testWordBreak() throws Exception {
assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = tokenFilterFactory("ThaiWord").create(stream);
assertTokenStreamContents(stream, new String[] {"การ", "ที่", "ได้",
"ต้อง", "แสดง", "ว่า", "งาน", "ดี"});
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilter.java (working copy)
@@ -37,8 +37,7 @@
* Test composed forms
*/
public void testTurkishLowerCaseFilter() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0130STANBUL \u0130ZM\u0130R ISPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0130STANBUL \u0130ZM\u0130R ISPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"istanbul", "izmir",
"\u0131sparta",});
@@ -48,8 +47,7 @@
* Test decomposed forms
*/
public void testDecomposed() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0307STANBUL \u0049\u0307ZM\u0049\u0307R ISPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0307STANBUL \u0049\u0307ZM\u0049\u0307R ISPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"istanbul", "izmir",
"\u0131sparta",});
@@ -61,16 +59,14 @@
* to U+0130 + U+0316, and is lowercased the same way.
*/
public void testDecomposed2() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0316\u0307STANBUL \u0049\u0307ZM\u0049\u0307R I\u0316SPARTA"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0316\u0307STANBUL \u0049\u0307ZM\u0049\u0307R I\u0316SPARTA");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"i\u0316stanbul", "izmir",
"\u0131\u0316sparta",});
}
public void testDecomposed3() throws Exception {
- TokenStream stream = new MockTokenizer(new StringReader(
- "\u0049\u0307"), MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer("\u0049\u0307");
TurkishLowerCaseFilter filter = new TurkishLowerCaseFilter(stream);
assertTokenStreamContents(filter, new String[] {"i"});
}
@@ -78,8 +74,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new TurkishLowerCaseFilter(tokenizer));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/tr/TestTurkishLowerCaseFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.BaseTokenStreamFactoryTestCase;
/**
@@ -33,7 +34,8 @@
*/
public void testCasing() throws Exception {
Reader reader = new StringReader("AĞACI");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("TurkishLowerCase").create(stream);
assertTokenStreamContents(stream, new String[] { "ağacı" });
}
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharTokenizers.java (working copy)
@@ -53,7 +53,8 @@
}
// internal buffer size is 1024 make sure we have a surrogate pair right at the border
builder.insert(1023, "\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, builder.toString().toLowerCase(Locale.ROOT).split(" "));
}
@@ -70,7 +71,8 @@
builder.append("a");
}
builder.append("\ud801\udc1cabc");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT)});
}
}
@@ -84,7 +86,8 @@
for (int i = 0; i < 255; i++) {
builder.append("A");
}
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -98,7 +101,8 @@
builder.append("A");
}
builder.append("\ud801\udc1c");
- Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ Tokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(builder.toString() + builder.toString()));
assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(Locale.ROOT), builder.toString().toLowerCase(Locale.ROOT)});
}
@@ -106,8 +110,8 @@
public void testCrossPlaneNormalization() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
@Override
protected int normalize(int c) {
if (c > 0xffff) {
@@ -144,8 +148,8 @@
public void testCrossPlaneNormalization2() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT, reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new LetterTokenizer(TEST_VERSION_CURRENT) {
@Override
protected int normalize(int c) {
if (c <= 0xffff) {
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElision.java (working copy)
@@ -40,7 +40,8 @@
public void testElision() throws Exception {
String test = "Plop, juste pour voir l'embrouille avec O'brian. M'enfin.";
- Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(test));
+ Tokenizer tokenizer = new StandardTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(test));
CharArraySet articles = new CharArraySet(TEST_VERSION_CURRENT, asSet("l", "M"), false);
TokenFilter filter = new ElisionFilter(tokenizer, articles);
List tas = filter(filter);
@@ -64,8 +65,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ElisionFilter(tokenizer, FrenchAnalyzer.DEFAULT_ARTICLES));
}
};
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestElisionFilterFactory.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Simple tests to ensure the French elision filter factory is working.
@@ -32,7 +33,8 @@
*/
public void testElision() throws Exception {
Reader reader = new StringReader("l'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision", "articles", "frenchArticles.txt").create(stream);
assertTokenStreamContents(stream, new String[] { "avion" });
}
@@ -42,7 +44,8 @@
*/
public void testDefaultArticles() throws Exception {
Reader reader = new StringReader("l'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision").create(stream);
assertTokenStreamContents(stream, new String[] { "avion" });
}
@@ -52,7 +55,8 @@
*/
public void testCaseInsensitive() throws Exception {
Reader reader = new StringReader("L'avion");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer)stream).setReader(reader);
stream = tokenFilterFactory("Elision",
"articles", "frenchArticles.txt",
"ignoreCase", "true").create(stream);
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/TestWikipediaTokenizerFactory.java (working copy)
@@ -30,7 +30,8 @@
public class TestWikipediaTokenizerFactory extends BaseTokenStreamFactoryTestCase {
public void testTokenizer() throws Exception {
Reader reader = new StringReader("This is a [[Category:foo]]");
- Tokenizer tokenizer = tokenizerFactory("Wikipedia").create(reader);
+ Tokenizer tokenizer = tokenizerFactory("Wikipedia").create();
+ tokenizer.setReader(reader);
assertTokenStreamContents(tokenizer,
new String[] { "This", "is", "a", "foo" },
new int[] { 0, 5, 8, 21 },
Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java
===================================================================
--- lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (revision 1556706)
+++ lucene/analysis/common/src/test/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerTest.java (working copy)
@@ -18,7 +18,6 @@
package org.apache.lucene.analysis.wikipedia;
-import java.io.Reader;
import java.io.StringReader;
import java.io.IOException;
import java.util.Random;
@@ -40,7 +39,8 @@
public void testSimple() throws Exception {
String text = "This is a [[Category:foo]]";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(text));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(text));
assertTokenStreamContents(tf,
new String[] { "This", "is", "a", "foo" },
new int[] { 0, 5, 8, 21 },
@@ -62,7 +62,8 @@
+ " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
+ " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] Citation martian code";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] {"link", "This", "is", "a",
"foo", "Category", "This", "is", "a", "linked", "bar", "none",
@@ -103,7 +104,8 @@
}
public void testLinkPhrases() throws Exception {
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
}
@@ -116,7 +118,8 @@
public void testLinks() throws Exception {
String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
+ WikipediaTokenizer tf = new WikipediaTokenizer();
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
"http://lucene.apache.org/java/docs/index.html?b=c", "here",
@@ -131,10 +134,12 @@
untoks.add(WikipediaTokenizer.CATEGORY);
untoks.add(WikipediaTokenizer.ITALICS);
//should be exactly the same, regardless of untoks
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES), WikipediaTokenizer.TOKENS_ONLY, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.TOKENS_ONLY, untoks);
+ tf.setReader(new StringReader(LINK_PHRASES));
checkLinkPhrases(tf);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
- tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
+ tf = new WikipediaTokenizer(WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "a b c d", "e f g", "link", "here", "link",
"there", "italics here", "something", "more italics", "h i j" },
@@ -150,7 +155,8 @@
untoks.add(WikipediaTokenizer.ITALICS);
String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
//should output all the indivual tokens plus the untokenized tokens as well. Untokenized tokens
- WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
+ WikipediaTokenizer tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ tf.setReader(new StringReader(test));
assertTokenStreamContents(tf,
new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
"link", "here", "link", "there", "italics here", "italics", "here",
@@ -161,7 +167,8 @@
);
// now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
- tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
+ tf = new WikipediaTokenizer(WikipediaTokenizer.BOTH, untoks);
+ tf.setReader(new StringReader(test));
int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
@@ -179,8 +186,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WikipediaTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WikipediaTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -193,8 +200,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new WikipediaTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new WikipediaTokenizer();
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java (revision 1556706)
+++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizer.java (working copy)
@@ -64,11 +64,10 @@
*
* The default attribute factory is used.
*
- * @param input Reader containing text to tokenize.
* @see DefaultICUTokenizerConfig
*/
- public ICUTokenizer(Reader input) {
- this(input, new DefaultICUTokenizerConfig(true));
+ public ICUTokenizer() {
+ this(new DefaultICUTokenizerConfig(true));
}
/**
@@ -77,11 +76,10 @@
*
* The default attribute factory is used.
*
- * @param input Reader containing text to tokenize.
- * @param config Tailored BreakIterator configuration
+ * @param config Tailored BreakIterator configuration
*/
- public ICUTokenizer(Reader input, ICUTokenizerConfig config) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, config);
+ public ICUTokenizer(ICUTokenizerConfig config) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, config);
}
/**
@@ -89,11 +87,10 @@
* Reader, using a tailored BreakIterator configuration.
*
* @param factory AttributeFactory to use
- * @param input Reader containing text to tokenize.
- * @param config Tailored BreakIterator configuration
+ * @param config Tailored BreakIterator configuration
*/
- public ICUTokenizer(AttributeFactory factory, Reader input, ICUTokenizerConfig config) {
- super(factory, input);
+ public ICUTokenizer(AttributeFactory factory, ICUTokenizerConfig config) {
+ super(factory);
this.config = config;
breaker = new CompositeBreakIterator(config);
}
Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (working copy)
@@ -144,8 +144,8 @@
}
@Override
- public ICUTokenizer create(AttributeFactory factory, Reader input) {
+ public ICUTokenizer create(AttributeFactory factory) {
assert config != null : "inform must be called first!";
- return new ICUTokenizer(factory, input, config);
+ return new ICUTokenizer(factory, config);
}
}
Index: lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (revision 1556706)
+++ lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (working copy)
@@ -81,8 +81,8 @@
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- KeywordTokenizer tokenizer = new KeywordTokenizer(factory, reader, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ KeywordTokenizer tokenizer = new KeywordTokenizer(factory, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
return new TokenStreamComponents(tokenizer, tokenizer);
}
}
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilter.java (working copy)
@@ -29,8 +29,8 @@
public class TestICUFoldingFilter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
}
};
@@ -82,8 +82,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUFoldingFilter(tokenizer));
}
};
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUFoldingFilterFactory.java (working copy)
@@ -32,7 +32,7 @@
public void test() throws Exception {
Reader reader = new StringReader("Résumé");
ICUFoldingFilterFactory factory = new ICUFoldingFilterFactory(new HashMap());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "resume" });
}
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2Filter.java (working copy)
@@ -31,8 +31,8 @@
public class TestICUNormalizer2Filter extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
}
};
@@ -61,8 +61,8 @@
public void testAlternate() throws IOException {
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(
tokenizer,
/* specify nfc with decompose to get nfd */
@@ -82,8 +82,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUNormalizer2Filter(tokenizer));
}
};
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUNormalizer2FilterFactory.java (working copy)
@@ -32,7 +32,7 @@
public void testDefaults() throws Exception {
Reader reader = new StringReader("This is a Ｔｅｓｔ");
ICUNormalizer2FilterFactory factory = new ICUNormalizer2FilterFactory(new HashMap());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "this", "is", "a", "test" });
}
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (working copy)
@@ -66,7 +66,9 @@
String rules = "a > b; b > c;"; // convert a's to b's and b's to c's
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
assertTrue(custom.getFilter() == null);
- new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
+ final KeywordTokenizer input = new KeywordTokenizer();
+ input.setReader(new StringReader(""));
+ new ICUTransformFilter(input, custom);
assertTrue(custom.getFilter().equals(new UnicodeSet("[ab]")));
}
@@ -79,12 +81,16 @@
String rules = "\\U00020087 > x;"; // convert CJK UNIFIED IDEOGRAPH-20087 to an x
Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD);
assertTrue(custom.getFilter() == null);
- new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom);
+ final KeywordTokenizer input = new KeywordTokenizer();
+ input.setReader(new StringReader(""));
+ new ICUTransformFilter(input, custom);
assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]")));
}
private void checkToken(Transliterator transform, String input, String expected) throws IOException {
- TokenStream ts = new ICUTransformFilter(new KeywordTokenizer((new StringReader(input))), transform);
+ final KeywordTokenizer input1 = new KeywordTokenizer();
+ input1.setReader(new StringReader(input));
+ TokenStream ts = new ICUTransformFilter(input1, transform);
assertTokenStreamContents(ts, new String[] { expected });
}
@@ -93,8 +99,8 @@
final Transliterator transform = Transliterator.getInstance("Any-Latin");
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, transform));
}
};
@@ -104,8 +110,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ICUTransformFilter(tokenizer, Transliterator.getInstance("Any-Latin")));
}
};
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilterFactory.java (working copy)
@@ -35,7 +35,7 @@
Map args = new HashMap();
args.put("id", "Traditional-Simplified");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "简化字" });
}
@@ -47,7 +47,7 @@
Map args = new HashMap();
args.put("id", "Cyrillic-Latin");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "Rossijskaâ", "Federaciâ" });
}
@@ -59,7 +59,7 @@
args.put("id", "Cyrillic-Latin");
args.put("direction", "reverse");
ICUTransformFilterFactory factory = new ICUTransformFilterFactory(args);
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] { "Российская", "Федерация" });
}
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (working copy)
@@ -42,7 +42,8 @@
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ tokenizer.setReader(new StringReader(input));
assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
@@ -52,7 +53,8 @@
sb.append('a');
}
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
+ ICUTokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
+ tokenizer.setReader(new StringReader(input));
char token[] = new char[4096];
Arrays.fill(token, 'a');
String expectedToken = new String(token);
@@ -67,9 +69,8 @@
private Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer tokenizer = new ICUTokenizer(reader, new DefaultICUTokenizerConfig(false));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerCJK.java (working copy)
@@ -29,8 +29,8 @@
public class TestICUTokenizerCJK extends BaseTokenStreamTestCase {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new ICUTokenizer(reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new ICUTokenizer());
}
};
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (working copy)
@@ -24,6 +24,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.util.ClasspathResourceLoader;
/** basic tests for {@link ICUTokenizerFactory} **/
@@ -32,7 +33,8 @@
Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี This is a test ກວ່າດອກ");
ICUTokenizerFactory factory = new ICUTokenizerFactory(new HashMap());
factory.inform(new ClasspathResourceLoader(getClass()));
- TokenStream stream = factory.create(reader);
+ Tokenizer stream = factory.create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
"This", "is", "a", "test", "ກວ່າ", "ດອກ"});
@@ -46,7 +48,8 @@
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(this.getClass()));
- TokenStream stream = factory.create(reader);
+ Tokenizer stream = factory.create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"", "+2=3$5,&813", "!@#%$^)(*@#$" },
new String[] { "", "", "", "", "", "" });
@@ -59,7 +62,8 @@
args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
- TokenStream stream = factory.create(reader);
+ Tokenizer stream = factory.create();
+ stream.setReader(reader);
assertTokenStreamContents(stream,
new String[] { "One-two", "punch",
"Brang", "not", "brung-it",
@@ -78,7 +82,8 @@
args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
ICUTokenizerFactory factory = new ICUTokenizerFactory(args);
factory.inform(new ClasspathResourceLoader(getClass()));
- TokenStream stream = factory.create(reader);
+ Tokenizer stream = factory.create();
+ stream.setReader(reader);
assertTokenStreamContents(stream, new String[] { "Some", "English",
"Немного русский. ",
"ข้อความภาษาไทยเล็ก ๆ น้อย ๆ ",
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java (revision 1556706)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestWithCJKBigramFilter.java (working copy)
@@ -40,8 +40,8 @@
*/
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new ICUTokenizer(reader, new DefaultICUTokenizerConfig(false));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
TokenStream result = new CJKBigramFilter(source);
return new TokenStreamComponents(source, new StopFilter(TEST_VERSION_CURRENT, result, CharArraySet.EMPTY_SET));
}
@@ -55,8 +55,8 @@
*/
private Analyzer analyzer2 = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new ICUTokenizer(reader, new DefaultICUTokenizerConfig(false));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new ICUTokenizer(new DefaultICUTokenizerConfig(false));
// we put this before the CJKBigramFilter, because the normalization might combine
// some halfwidth katakana forms, which will affect the bigramming.
TokenStream result = new ICUNormalizer2Filter(source);
Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java (working copy)
@@ -86,8 +86,8 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, userDict, true, mode);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(userDict, true, mode);
TokenStream stream = new JapaneseBaseFormFilter(tokenizer);
stream = new JapanesePartOfSpeechStopFilter(matchVersion, stream, stoptags);
stream = new CJKWidthFilter(stream);
Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizer.java (working copy)
@@ -190,27 +190,25 @@
*
* Uses the default AttributeFactory.
*
- * @param input Reader containing text
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
* @param mode tokenization mode.
*/
- public JapaneseTokenizer(Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, userDictionary, discardPunctuation, mode);
+ public JapaneseTokenizer(UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, userDictionary, discardPunctuation, mode);
}
/**
* Create a new JapaneseTokenizer.
*
* @param factory the AttributeFactory to use
- * @param input Reader containing text
* @param userDictionary Optional: if non-null, user dictionary.
* @param discardPunctuation true if punctuation tokens should be dropped from the output.
* @param mode tokenization mode.
*/
public JapaneseTokenizer
- (AttributeFactory factory, Reader input, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
- super(factory, input);
+ (AttributeFactory factory, UserDictionary userDictionary, boolean discardPunctuation, Mode mode) {
+ super(factory);
dictionary = TokenInfoDictionary.getInstance();
fst = dictionary.getFST();
unkDictionary = UnknownDictionary.getInstance();
Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (working copy)
@@ -98,7 +98,7 @@
}
@Override
- public JapaneseTokenizer create(AttributeFactory factory, Reader input) {
- return new JapaneseTokenizer(factory, input, userDictionary, discardPunctuation, mode);
+ public JapaneseTokenizer create(AttributeFactory factory) {
+ return new JapaneseTokenizer(factory, userDictionary, discardPunctuation, mode);
}
}
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestExtendedMode.java (working copy)
@@ -36,8 +36,8 @@
private final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, Mode.EXTENDED);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilter.java (working copy)
@@ -31,8 +31,8 @@
public class TestJapaneseBaseFormFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.DEFAULT_MODE);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
}
};
@@ -47,8 +47,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("あり"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.DEFAULT_MODE);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new JapaneseTokenizer(null, true, JapaneseTokenizer.DEFAULT_MODE);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new JapaneseBaseFormFilter(sink));
}
@@ -70,8 +70,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new JapaneseBaseFormFilter(tokenizer));
}
};
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseBaseFormFilterFactory.java (working copy)
@@ -23,6 +23,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Simple tests for {@link JapaneseBaseFormFilterFactory}
@@ -31,7 +32,8 @@
public void testBasics() throws IOException {
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream ts = tokenizerFactory.create(new StringReader("それはまだ実験段階にあります"));
+ TokenStream ts = tokenizerFactory.create();
+ ((Tokenizer)ts).setReader(new StringReader("それはまだ実験段階にあります"));
JapaneseBaseFormFilterFactory factory = new JapaneseBaseFormFilterFactory(new HashMap());
ts = factory.create(ts);
assertTokenStreamContents(ts,
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilter.java (working copy)
@@ -31,8 +31,8 @@
private Analyzer keywordAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.KEYWORD, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
@@ -44,8 +44,8 @@
private Analyzer japaneseAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, false, JapaneseTokenizer.Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, false, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseIterationMarkCharFilterFactory.java (working copy)
@@ -21,6 +21,7 @@
import org.apache.lucene.analysis.CharFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import java.io.IOException;
import java.io.StringReader;
@@ -36,7 +37,8 @@
final String text = "時々馬鹿々々しいところゞゝゝミスヾ";
JapaneseIterationMarkCharFilterFactory filterFactory = new JapaneseIterationMarkCharFilterFactory(new HashMap());
CharFilter filter = filterFactory.create(new StringReader(text));
- TokenStream tokenStream = new MockTokenizer(filter, MockTokenizer.KEYWORD, false);
+ TokenStream tokenStream = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ ((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"時時馬鹿馬鹿しいところどころミスズ"});
}
@@ -48,7 +50,8 @@
CharFilter filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
- TokenStream tokenStream = tokenizerFactory.create(filter);
+ TokenStream tokenStream = tokenizerFactory.create();
+ ((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところどころ", "ミ", "スズ"});
}
@@ -64,7 +67,8 @@
CharFilter filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
- TokenStream tokenStream = tokenizerFactory.create(filter);
+ TokenStream tokenStream = tokenizerFactory.create();
+ ((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"時時", "馬鹿馬鹿しい", "ところ", "ゞ", "ゝ", "ゝ", "ミス", "ヾ"});
}
@@ -80,7 +84,8 @@
CharFilter filter = filterFactory.create(
new StringReader("時々馬鹿々々しいところゞゝゝミスヾ")
);
- TokenStream tokenStream = tokenizerFactory.create(filter);
+ TokenStream tokenStream = tokenizerFactory.create();
+ ((Tokenizer)tokenStream).setReader(filter);
assertTokenStreamContents(tokenStream, new String[]{"時々", "馬鹿", "々", "々", "しい", "ところどころ", "ミ", "スズ"});
}
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilter.java (working copy)
@@ -35,9 +35,9 @@
public class TestJapaneseKatakanaStemFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
// Use a MockTokenizer here since this filter doesn't really depend on Kuromoji
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(source));
}
};
@@ -68,8 +68,8 @@
final CharArraySet exclusionSet = new CharArraySet(TEST_VERSION_CURRENT, asSet("コーヒー"), false);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream sink = new SetKeywordMarkerFilter(source, exclusionSet);
return new TokenStreamComponents(source, new JapaneseKatakanaStemFilter(sink));
}
@@ -89,8 +89,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new JapaneseKatakanaStemFilter(tokenizer));
}
};
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseKatakanaStemFilterFactory.java (working copy)
@@ -19,6 +19,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import java.io.IOException;
import java.io.StringReader;
@@ -31,9 +32,8 @@
public void testKatakanaStemming() throws IOException {
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream tokenStream = tokenizerFactory.create(
- new StringReader("明後日パーティーに行く予定がある。図書館で資料をコピーしました。")
- );
+ TokenStream tokenStream = tokenizerFactory.create();
+ ((Tokenizer)tokenStream).setReader(new StringReader("明後日パーティーに行く予定がある。図書館で資料をコピーしました。"));
JapaneseKatakanaStemFilterFactory filterFactory = new JapaneseKatakanaStemFilterFactory(new HashMap());;
assertTokenStreamContents(filterFactory.create(tokenStream),
new String[]{ "明後日", "パーティ", "に", "行く", "予定", "が", "ある", // パーティー should be stemmed
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapanesePartOfSpeechStopFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapanesePartOfSpeechStopFilterFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapanesePartOfSpeechStopFilterFactory.java (working copy)
@@ -24,6 +24,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Simple tests for {@link JapanesePartOfSpeechStopFilterFactory}
@@ -36,7 +37,8 @@
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream ts = tokenizerFactory.create(new StringReader("私は制限スピードを超える。"));
+ TokenStream ts = tokenizerFactory.create();
+ ((Tokenizer)ts).setReader(new StringReader("私は制限スピードを超える。"));
Map args = new HashMap();
args.put("luceneMatchVersion", TEST_VERSION_CURRENT.toString());
args.put("tags", "stoptags.txt");
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilter.java (working copy)
@@ -34,16 +34,16 @@
public class TestJapaneseReadingFormFilter extends BaseTokenStreamTestCase {
private Analyzer katakanaAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, false));
}
};
private Analyzer romajiAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer, true));
}
};
@@ -58,8 +58,8 @@
public void testKatakanaReadingsHalfWidth() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
TokenStream stream = new CJKWidthFilter(tokenizer);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, false));
}
@@ -78,8 +78,8 @@
public void testRomajiReadingsHalfWidth() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, JapaneseTokenizer.Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, JapaneseTokenizer.Mode.SEARCH);
TokenStream stream = new CJKWidthFilter(tokenizer);
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(stream, true));
}
@@ -98,8 +98,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new JapaneseReadingFormFilter(tokenizer));
}
};
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilterFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseReadingFormFilterFactory.java (working copy)
@@ -19,6 +19,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import java.io.IOException;
import java.io.StringReader;
@@ -31,7 +32,8 @@
public void testReadings() throws IOException {
JapaneseTokenizerFactory tokenizerFactory = new JapaneseTokenizerFactory(new HashMap());
tokenizerFactory.inform(new StringMockResourceLoader(""));
- TokenStream tokenStream = tokenizerFactory.create(new StringReader("先ほどベルリンから来ました。"));
+ TokenStream tokenStream = tokenizerFactory.create();
+ ((Tokenizer)tokenStream).setReader(new StringReader("先ほどベルリンから来ました。"));
JapaneseReadingFormFilterFactory filterFactory = new JapaneseReadingFormFilterFactory(new HashMap());
assertTokenStreamContents(filterFactory.create(tokenStream),
new String[] { "サキ", "ホド", "ベルリン", "カラ", "キ", "マシ", "タ" }
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizer.java (working copy)
@@ -61,32 +61,32 @@
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), false, Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
private Analyzer analyzerNormal = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), false, Mode.NORMAL);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.NORMAL);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
private Analyzer analyzerNoPunct = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), true, Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), true, Mode.EXTENDED);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(readDict(), true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
@@ -201,8 +201,8 @@
checkRandomData(random,
new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), false, Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
TokenStream graph = new MockGraphTokenFilter(random(), tokenizer);
return new TokenStreamComponents(tokenizer, graph);
}
@@ -351,8 +351,8 @@
final GraphvizFormatter gv2 = new GraphvizFormatter(ConnectionCosts.getInstance());
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- JapaneseTokenizer tokenizer = new JapaneseTokenizer(reader, readDict(), false, Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ JapaneseTokenizer tokenizer = new JapaneseTokenizer(readDict(), false, Mode.SEARCH);
tokenizer.setGraphvizFormatter(gv2);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseTokenizerFactory.java (working copy)
@@ -25,6 +25,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Simple tests for {@link JapaneseTokenizerFactory}
@@ -33,7 +34,8 @@
public void testSimple() throws IOException {
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap());
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create(new StringReader("これは本ではない"));
+ TokenStream ts = factory.create();
+ ((Tokenizer)ts).setReader(new StringReader("これは本ではない"));
assertTokenStreamContents(ts,
new String[] { "これ", "は", "本", "で", "は", "ない" },
new int[] { 0, 2, 3, 4, 5, 6 },
@@ -47,7 +49,8 @@
public void testDefaults() throws IOException {
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(new HashMap());
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create(new StringReader("シニアソフトウェアエンジニア"));
+ TokenStream ts = factory.create();
+ ((Tokenizer)ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
assertTokenStreamContents(ts,
new String[] { "シニア", "シニアソフトウェアエンジニア", "ソフトウェア", "エンジニア" }
);
@@ -61,7 +64,8 @@
args.put("mode", "normal");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create(new StringReader("シニアソフトウェアエンジニア"));
+ TokenStream ts = factory.create();
+ ((Tokenizer)ts).setReader(new StringReader("シニアソフトウェアエンジニア"));
assertTokenStreamContents(ts,
new String[] { "シニアソフトウェアエンジニア" }
);
@@ -81,7 +85,8 @@
args.put("userDictionary", "userdict.txt");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(userDict));
- TokenStream ts = factory.create(new StringReader("関西国際空港に行った"));
+ TokenStream ts = factory.create();
+ ((Tokenizer)ts).setReader(new StringReader("関西国際空港に行った"));
assertTokenStreamContents(ts,
new String[] { "関西", "国際", "空港", "に", "行っ", "た" }
);
@@ -95,9 +100,8 @@
args.put("discardPunctuation", "false");
JapaneseTokenizerFactory factory = new JapaneseTokenizerFactory(args);
factory.inform(new StringMockResourceLoader(""));
- TokenStream ts = factory.create(
- new StringReader("今ノルウェーにいますが、来週の頭日本に戻ります。楽しみにしています！お寿司が食べたいな。。。")
- );
+ TokenStream ts = factory.create();
+ ((Tokenizer)ts).setReader(new StringReader("今ノルウェーにいますが、来週の頭日本に戻ります。楽しみにしています！お寿司が食べたいな。。。"));
assertTokenStreamContents(ts,
new String[] { "今", "ノルウェー", "に", "い", "ます", "が", "、",
"来週", "の", "頭", "日本", "に", "戻り", "ます", "。",
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java (revision 1556706)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestSearchMode.java (working copy)
@@ -34,8 +34,8 @@
private final static String SEGMENTATION_FILENAME = "search-segmentation-tests.txt";
private final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new JapaneseTokenizer(reader, null, true, Mode.SEARCH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new JapaneseTokenizer(null, true, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java
===================================================================
--- lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java (revision 1556706)
+++ lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/morfologik/MorfologikAnalyzer.java (working copy)
@@ -49,16 +49,14 @@
* which tokenizes all the text in the provided {@link Reader}.
*
* @param field ignored field name
- * @param reader source of tokens
- *
* @return A
* {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents}
* built from an {@link StandardTokenizer} filtered with
* {@link StandardFilter} and {@link MorfologikFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(final String field, final Reader reader) {
- final Tokenizer src = new StandardTokenizer(this.version, reader);
+ protected TokenStreamComponents createComponents(final String field) {
+ final Tokenizer src = new StandardTokenizer(this.version);
return new TokenStreamComponents(
src,
Index: lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java
===================================================================
--- lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java (revision 1556706)
+++ lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java (working copy)
@@ -171,11 +171,11 @@
Analyzer a = new MorfologikAnalyzer(version) {
@Override
- protected TokenStreamComponents createComponents(String field, Reader reader) {
+ protected TokenStreamComponents createComponents(String field) {
final CharArraySet keywords = new CharArraySet(version, 1, false);
keywords.add("liście");
- final Tokenizer src = new StandardTokenizer(TEST_VERSION_CURRENT, reader);
+ final Tokenizer src = new StandardTokenizer(TEST_VERSION_CURRENT);
TokenStream result = new StandardFilter(TEST_VERSION_CURRENT, src);
result = new SetKeywordMarkerFilter(result, keywords);
result = new MorfologikFilter(result, TEST_VERSION_CURRENT);
Index: lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java
===================================================================
--- lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java (revision 1556706)
+++ lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java (working copy)
@@ -32,7 +32,7 @@
public void testCreateDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety");
MorfologikFilterFactory factory = new MorfologikFilterFactory(Collections.emptyMap());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream, new String[] {"rower", "bilet"});
}
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/DoubleMetaphoneFilterTest.java (working copy)
@@ -30,44 +30,50 @@
import org.apache.lucene.util._TestUtil;
public class DoubleMetaphoneFilterTest extends BaseTokenStreamTestCase {
+
+ private TokenStream whitespaceTokenizer(String data) throws IOException {
+ WhitespaceTokenizer whitespaceTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ whitespaceTokenizer.setReader(new StringReader(data));
+ return whitespaceTokenizer;
+ }
public void testSize4FalseInject() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("international"));
+ TokenStream stream = whitespaceTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
assertTokenStreamContents(filter, new String[] { "ANTR" });
}
public void testSize4TrueInject() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("international"));
+ TokenStream stream = whitespaceTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, true);
assertTokenStreamContents(filter, new String[] { "international", "ANTR" });
}
public void testAlternateInjectFalse() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("Kuczewski"));
+ TokenStream stream = whitespaceTokenizer("Kuczewski");
TokenStream filter = new DoubleMetaphoneFilter(stream, 4, false);
assertTokenStreamContents(filter, new String[] { "KSSK", "KXFS" });
}
public void testSize8FalseInject() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("international"));
+ TokenStream stream = whitespaceTokenizer("international");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "ANTRNXNL" });
}
public void testNonConvertableStringsWithInject() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("12345 #$%@#^%&"));
+ TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, true);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
}
public void testNonConvertableStringsWithoutInject() throws Exception {
- TokenStream stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("12345 #$%@#^%&"));
+ TokenStream stream = whitespaceTokenizer("12345 #$%@#^%&");
TokenStream filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&" });
// should have something after the stream
- stream = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("12345 #$%@#^%& hello"));
+ stream = whitespaceTokenizer("12345 #$%@#^%& hello");
filter = new DoubleMetaphoneFilter(stream, 8, false);
assertTokenStreamContents(filter, new String[] { "12345", "#$%@#^%&", "HL" });
}
@@ -77,8 +83,8 @@
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, false));
}
@@ -88,8 +94,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, codeLen, true));
}
@@ -100,8 +106,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new DoubleMetaphoneFilter(tokenizer, 8, random().nextBoolean()));
}
};
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilter.java (working copy)
@@ -41,8 +41,8 @@
public class TestBeiderMorseFilter extends BaseTokenStreamTestCase {
private Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new BeiderMorseFilter(tokenizer, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true)));
}
@@ -71,8 +71,8 @@
}});
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer( MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer,
new BeiderMorseFilter(tokenizer,
new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true), languages));
@@ -101,8 +101,8 @@
public void testEmptyTerm() throws IOException {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new BeiderMorseFilter(tokenizer, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true)));
}
};
@@ -110,7 +110,8 @@
}
public void testCustomAttribute() throws IOException {
- TokenStream stream = new KeywordTokenizer(new StringReader("D'Angelo"));
+ TokenStream stream = new KeywordTokenizer();
+ ((Tokenizer)stream).setReader(new StringReader("D'Angelo"));
stream = new PatternKeywordMarkerFilter(stream, Pattern.compile(".*"));
stream = new BeiderMorseFilter(stream, new PhoneticEngine(NameType.GENERIC, RuleType.EXACT, true));
KeywordAttribute keyAtt = stream.addAttribute(KeywordAttribute.class);
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilterFactory.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilterFactory.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestBeiderMorseFilterFactory.java (working copy)
@@ -29,7 +29,7 @@
public class TestBeiderMorseFilterFactory extends BaseTokenStreamTestCase {
public void testBasics() throws Exception {
BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(new HashMap());
- TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
+ TokenStream ts = factory.create(whitespaceMockTokenizer("Weinberg"));
assertTokenStreamContents(ts,
new String[] { "vDnbirk", "vanbirk", "vinbirk", "wDnbirk", "wanbirk", "winbirk" },
new int[] { 0, 0, 0, 0, 0, 0 },
@@ -41,7 +41,7 @@
Map args = new HashMap();
args.put("languageSet", "polish");
BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
- TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
+ TokenStream ts = factory.create(whitespaceMockTokenizer("Weinberg"));
assertTokenStreamContents(ts,
new String[] { "vDmbYrk", "vDmbirk", "vambYrk", "vambirk", "vimbYrk", "vimbirk" },
new int[] { 0, 0, 0, 0, 0, 0 },
@@ -54,7 +54,7 @@
args.put("nameType", "ASHKENAZI");
args.put("ruleType", "EXACT");
BeiderMorseFilterFactory factory = new BeiderMorseFilterFactory(args);
- TokenStream ts = factory.create(new MockTokenizer(new StringReader("Weinberg"), MockTokenizer.WHITESPACE, false));
+ TokenStream ts = factory.create(whitespaceMockTokenizer("Weinberg"));
assertTokenStreamContents(ts,
new String[] { "vajnberk" },
new int[] { 0 },
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDoubleMetaphoneFilterFactory.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDoubleMetaphoneFilterFactory.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestDoubleMetaphoneFilterFactory.java (working copy)
@@ -31,7 +31,7 @@
public void testDefaults() throws Exception {
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(new HashMap());
- TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
+ TokenStream inputStream = whitespaceMockTokenizer("international");
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
@@ -44,7 +44,7 @@
parameters.put("maxCodeLength", "8");
DoubleMetaphoneFilterFactory factory = new DoubleMetaphoneFilterFactory(parameters);
- TokenStream inputStream = new MockTokenizer(new StringReader("international"), MockTokenizer.WHITESPACE, false);
+ TokenStream inputStream = whitespaceMockTokenizer("international");
TokenStream filteredStream = factory.create(inputStream);
assertEquals(DoubleMetaphoneFilter.class, filteredStream.getClass());
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilter.java (working copy)
@@ -66,8 +66,8 @@
static void assertAlgorithm(Encoder encoder, boolean inject, String input,
String[] expected) throws Exception {
- Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader(input));
+ Tokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT);
+ tokenizer.setReader(new StringReader(input));
PhoneticFilter filter = new PhoneticFilter(tokenizer, encoder, inject);
assertTokenStreamContents(filter, expected);
}
@@ -81,8 +81,8 @@
for (final Encoder e : encoders) {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
}
};
@@ -91,8 +91,8 @@
Analyzer b = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, false));
}
};
@@ -108,8 +108,8 @@
for (final Encoder e : encoders) {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new PhoneticFilter(tokenizer, e, random().nextBoolean()));
}
};
Index: lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilterFactory.java
===================================================================
--- lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilterFactory.java (revision 1556706)
+++ lucene/analysis/phonetic/src/test/org/apache/lucene/analysis/phonetic/TestPhoneticFilterFactory.java (working copy)
@@ -181,7 +181,7 @@
static void assertAlgorithm(String algName, String inject, String input,
String[] expected) throws Exception {
- Tokenizer tokenizer = new MockTokenizer(new StringReader(input), MockTokenizer.WHITESPACE, false);
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
Map args = new HashMap();
args.put("encoder", algName);
args.put("inject", inject);
Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
===================================================================
--- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (revision 1556706)
+++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (working copy)
@@ -48,12 +48,11 @@
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
- public SentenceTokenizer(Reader reader) {
- super(reader);
+ public SentenceTokenizer() {
}
- public SentenceTokenizer(AttributeFactory factory, Reader reader) {
- super(factory, reader);
+ public SentenceTokenizer(AttributeFactory factory) {
+ super(factory);
}
@Override
Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
===================================================================
--- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (revision 1556706)
+++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java (working copy)
@@ -136,8 +136,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new SentenceTokenizer(reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new SentenceTokenizer();
TokenStream result = new WordTokenFilter(tokenizer);
// result = new LowerCaseFilter(result);
// LowerCaseFilter is not needed, as SegTokenFilter lowercases Basic Latin text.
Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java
===================================================================
--- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (working copy)
@@ -38,7 +38,7 @@
}
@Override
- public SentenceTokenizer create(AttributeFactory factory, Reader input) {
- return new SentenceTokenizer(factory, input);
+ public SentenceTokenizer create(AttributeFactory factory) {
+ return new SentenceTokenizer(factory);
}
}
Index: lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
===================================================================
--- lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (revision 1556706)
+++ lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (working copy)
@@ -211,8 +211,8 @@
public void testInvalidOffset() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenFilter filters = new ASCIIFoldingFilter(tokenizer);
filters = new WordTokenFilter(filters);
return new TokenStreamComponents(tokenizer, filters);
@@ -240,8 +240,8 @@
Random random = random();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new WordTokenFilter(tokenizer));
}
};
Index: lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseFactories.java
===================================================================
--- lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseFactories.java (revision 1556706)
+++ lucene/analysis/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseFactories.java (working copy)
@@ -24,6 +24,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
/**
* Tests for {@link SmartChineseSentenceTokenizerFactory} and
@@ -33,7 +34,7 @@
/** Test showing the behavior with whitespace */
public void testSimple() throws Exception {
Reader reader = new StringReader("我购买了道具和服装。");
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
SmartChineseWordTokenFilterFactory factory = new SmartChineseWordTokenFilterFactory(new HashMap());
stream = factory.create(stream);
// TODO: fix smart chinese to not emit punctuation tokens
@@ -46,7 +47,8 @@
public void testTokenizer() throws Exception {
Reader reader = new StringReader("我购买了道具和服装。我购买了道具和服装。");
SmartChineseSentenceTokenizerFactory tokenizerFactory = new SmartChineseSentenceTokenizerFactory(new HashMap());
- TokenStream stream = tokenizerFactory.create(reader);
+ TokenStream stream = tokenizerFactory.create();
+ ((Tokenizer)stream).setReader(reader);
SmartChineseWordTokenFilterFactory factory = new SmartChineseWordTokenFilterFactory(new HashMap());
stream = factory.create(stream);
// TODO: fix smart chinese to not emit punctuation tokens
Index: lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
===================================================================
--- lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (revision 1556706)
+++ lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java (working copy)
@@ -139,9 +139,8 @@
* provided and {@link StempelFilter}.
*/
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- final Tokenizer source = new StandardTokenizer(matchVersion, reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer source = new StandardTokenizer(matchVersion);
TokenStream result = new StandardFilter(matchVersion, source);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
Index: lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java
===================================================================
--- lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java (revision 1556706)
+++ lucene/analysis/stempel/src/test/org/apache/lucene/analysis/stempel/TestStempelPolishStemFilterFactory.java (working copy)
@@ -32,7 +32,7 @@
public void testBasics() throws Exception {
Reader reader = new StringReader("studenta studenci");
StempelPolishStemFilterFactory factory = new StempelPolishStemFilterFactory(new HashMap());
- TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ TokenStream stream = whitespaceMockTokenizer(reader);
stream = factory.create(stream);
assertTokenStreamContents(stream,
new String[] { "student", "student" });
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/BaseUIMATokenizer.java (working copy)
@@ -45,8 +45,8 @@
protected CAS cas;
protected BaseUIMATokenizer
- (AttributeFactory factory, Reader reader, String descriptorPath, Map configurationParameters) {
- super(factory, reader);
+ (AttributeFactory factory, String descriptorPath, Map configurationParameters) {
+ super(factory);
this.descriptorPath = descriptorPath;
this.configurationParameters = configurationParameters;
}
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizer.java (working copy)
@@ -42,13 +42,13 @@
private int finalOffset = 0;
- public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Map configurationParameters, Reader input) {
- this(descriptorPath, tokenType, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
+ public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Map configurationParameters) {
+ this(descriptorPath, tokenType, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
}
public UIMAAnnotationsTokenizer(String descriptorPath, String tokenType, Map configurationParameters,
- AttributeFactory factory, Reader input) {
- super(factory, input, descriptorPath, configurationParameters);
+ AttributeFactory factory) {
+ super(factory, descriptorPath, configurationParameters);
this.tokenTypeString = tokenType;
this.termAttr = addAttribute(CharTermAttribute.class);
this.offsetAttr = addAttribute(OffsetAttribute.class);
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (working copy)
@@ -42,7 +42,7 @@
}
@Override
- public UIMAAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
- return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, factory, input);
+ public UIMAAnnotationsTokenizer create(AttributeFactory factory) {
+ return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, factory);
}
}
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMABaseAnalyzer.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMABaseAnalyzer.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMABaseAnalyzer.java (working copy)
@@ -38,8 +38,8 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters, reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new UIMAAnnotationsTokenizer(descriptorPath, tokenType, configurationParameters));
}
}
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzer.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzer.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnalyzer.java (working copy)
@@ -19,7 +19,6 @@
import org.apache.lucene.analysis.Analyzer;
-import java.io.Reader;
import java.util.Map;
/**
@@ -39,7 +38,7 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, configurationParameters, reader));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, configurationParameters));
}
}
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizer.java (working copy)
@@ -52,13 +52,13 @@
private int finalOffset = 0;
- public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath, Map configurationParameters, Reader input) {
- this(descriptorPath, tokenType, typeAttributeFeaturePath, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input);
+ public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath, Map configurationParameters) {
+ this(descriptorPath, tokenType, typeAttributeFeaturePath, configurationParameters, AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
}
public UIMATypeAwareAnnotationsTokenizer(String descriptorPath, String tokenType, String typeAttributeFeaturePath,
- Map configurationParameters, AttributeFactory factory, Reader input) {
- super(factory, input, descriptorPath, configurationParameters);
+ Map configurationParameters, AttributeFactory factory) {
+ super(factory, descriptorPath, configurationParameters);
this.tokenTypeString = tokenType;
this.termAttr = addAttribute(CharTermAttribute.class);
this.typeAttr = addAttribute(TypeAttribute.class);
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (revision 1556706)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (working copy)
@@ -44,8 +44,8 @@
}
@Override
- public UIMATypeAwareAnnotationsTokenizer create(AttributeFactory factory, Reader input) {
+ public UIMATypeAwareAnnotationsTokenizer create(AttributeFactory factory) {
return new UIMATypeAwareAnnotationsTokenizer
- (descriptorPath, tokenType, featurePath, configurationParameters, factory, input);
+ (descriptorPath, tokenType, featurePath, configurationParameters, factory);
}
}
Index: lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/AnalyzerFactory.java
===================================================================
--- lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/AnalyzerFactory.java (revision 1556706)
+++ lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/AnalyzerFactory.java (working copy)
@@ -78,8 +78,8 @@
}
@Override
- protected Analyzer.TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer tokenizer = tokenizerFactory.create(reader);
+ protected Analyzer.TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer tokenizer = tokenizerFactory.create();
TokenStream tokenStream = tokenizer;
for (TokenFilterFactory filterFactory : tokenFilterFactories) {
tokenStream = filterFactory.create(tokenStream);
Index: lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java
===================================================================
--- lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java (revision 1556706)
+++ lucene/classification/src/test/org/apache/lucene/classification/SimpleNaiveBayesClassifierTest.java (working copy)
@@ -55,8 +55,8 @@
private class NGramAnalyzer extends Analyzer {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer tokenizer = new KeywordTokenizer();
return new TokenStreamComponents(tokenizer, new ReverseStringFilter(TEST_VERSION_CURRENT, new EdgeNGramTokenFilter(TEST_VERSION_CURRENT, new ReverseStringFilter(TEST_VERSION_CURRENT, tokenizer), 10, 20)));
}
}
Index: lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (revision 1556706)
+++ lucene/core/src/java/org/apache/lucene/analysis/Analyzer.java (working copy)
@@ -31,14 +31,14 @@
* policy for extracting index terms from text.
*
* In order to define what analysis is done, subclasses must define their
- * {@link TokenStreamComponents TokenStreamComponents} in {@link #createComponents(String, Reader)}.
+ * {@link TokenStreamComponents TokenStreamComponents} in {@link #createComponents(String)}.
* The components are then reused in each call to {@link #tokenStream(String, Reader)}.
*
* Simple example:
*
* Analyzer analyzer = new Analyzer() {
* {@literal @Override}
- * protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ * protected TokenStreamComponents createComponents(String fieldName) {
* Tokenizer source = new FooTokenizer(reader);
* TokenStream filter = new FooFilter(source);
* filter = new BarFilter(filter);
@@ -101,18 +101,16 @@
* @param fieldName
* the name of the fields content passed to the
* {@link TokenStreamComponents} sink as a reader
- * @param reader
- * the reader passed to the {@link Tokenizer} constructor
+
* @return the {@link TokenStreamComponents} for this analyzer.
*/
- protected abstract TokenStreamComponents createComponents(String fieldName,
- Reader reader);
+ protected abstract TokenStreamComponents createComponents(String fieldName);
/**
* Returns a TokenStream suitable for fieldName, tokenizing
* the contents of reader.
*
- * This method uses {@link #createComponents(String, Reader)} to obtain an
+ * This method uses {@link #createComponents(String)} to obtain an
* instance of {@link TokenStreamComponents}. It returns the sink of the
* components and stores the components internally. Subsequent calls to this
* method will reuse the previously stored components after resetting them
@@ -139,11 +137,10 @@
TokenStreamComponents components = reuseStrategy.getReusableComponents(this, fieldName);
final Reader r = initReader(fieldName, reader);
if (components == null) {
- components = createComponents(fieldName, r);
+ components = createComponents(fieldName);
reuseStrategy.setReusableComponents(this, fieldName, components);
- } else {
- components.setReader(r);
}
+ components.setReader(r);
return components.getTokenStream();
}
@@ -151,7 +148,7 @@
* Returns a TokenStream suitable for fieldName, tokenizing
* the contents of text.
*
- * This method uses {@link #createComponents(String, Reader)} to obtain an
+ * This method uses {@link #createComponents(String)} to obtain an
* instance of {@link TokenStreamComponents}. It returns the sink of the
* components and stores the components internally. Subsequent calls to this
* method will reuse the previously stored components after resetting them
@@ -177,11 +174,11 @@
strReader.setValue(text);
final Reader r = initReader(fieldName, strReader);
if (components == null) {
- components = createComponents(fieldName, r);
+ components = createComponents(fieldName);
reuseStrategy.setReusableComponents(this, fieldName, components);
- } else {
- components.setReader(r);
}
+
+ components.setReader(r);
components.reusableStringReader = strReader;
return components.getTokenStream();
}
Index: lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java (revision 1556706)
+++ lucene/core/src/java/org/apache/lucene/analysis/AnalyzerWrapper.java (working copy)
@@ -98,8 +98,8 @@
}
@Override
- protected final TokenStreamComponents createComponents(String fieldName, Reader aReader) {
- return wrapComponents(fieldName, getWrappedAnalyzer(fieldName).createComponents(fieldName, aReader));
+ protected final TokenStreamComponents createComponents(String fieldName) {
+ return wrapComponents(fieldName, getWrappedAnalyzer(fieldName).createComponents(fieldName));
}
@Override
Index: lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (revision 1556706)
+++ lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (working copy)
@@ -37,21 +37,21 @@
/** Pending reader: not actually assigned to input until reset() */
private Reader inputPending = ILLEGAL_STATE_READER;
- /** Construct a token stream processing the given input. */
- protected Tokenizer(Reader input) {
- if (input == null) {
- throw new NullPointerException("input must not be null");
- }
- this.inputPending = input;
+ /**
+ * Construct a tokenizer with no input, awaiting a call to {@link #setReader(java.io.Reader)}
+ * to provide input.
+ */
+ protected Tokenizer() {
+ //
}
-
- /** Construct a token stream processing the given input using the given AttributeFactory. */
- protected Tokenizer(AttributeFactory factory, Reader input) {
+
+ /**
+ * Construct a tokenizer with no input, awaiting a call to {@link #setReader(java.io.Reader)} to
+ * provide input.
+ * @param factory attribute factory.
+ */
+ protected Tokenizer(AttributeFactory factory) {
super(factory);
- if (input == null) {
- throw new NullPointerException("input must not be null");
- }
- this.inputPending = input;
}
/**
Index: lucene/core/src/test/org/apache/lucene/analysis/TestGraphTokenizers.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestGraphTokenizers.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestGraphTokenizers.java (working copy)
@@ -17,13 +17,9 @@
* limitations under the License.
*/
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.Reader;
import java.io.StringWriter;
import java.io.PrintWriter;
-import java.io.Writer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -63,10 +59,6 @@
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final PositionLengthAttribute posLengthAtt = addAttribute(PositionLengthAttribute.class);
- public GraphTokenizer(Reader input) {
- super(input);
- }
-
@Override
public void reset() throws IOException {
super.reset();
@@ -174,8 +166,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t2 = new MockGraphTokenFilter(random(), t);
return new TokenStreamComponents(t, t2);
}
@@ -196,8 +188,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new GraphTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new GraphTokenizer();
final TokenStream t2 = new MockGraphTokenFilter(random(), t);
return new TokenStreamComponents(t, t2);
}
@@ -258,8 +250,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t2 = new MockGraphTokenFilter(random(), t);
final TokenStream t3 = new RemoveATokens(t2);
return new TokenStreamComponents(t, t3);
@@ -285,8 +277,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t2 = new RemoveATokens(t);
final TokenStream t3 = new MockGraphTokenFilter(random(), t2);
return new TokenStreamComponents(t, t3);
@@ -312,8 +304,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t2 = new MockGraphTokenFilter(random(), t);
return new TokenStreamComponents(t, t2);
}
@@ -336,8 +328,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t1 = new MockGraphTokenFilter(random(), t);
final TokenStream t2 = new MockGraphTokenFilter(random(), t1);
return new TokenStreamComponents(t, t2);
@@ -360,8 +352,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t1 = new MockGraphTokenFilter(random(), t);
final TokenStream t2 = new MockHoleInjectingTokenFilter(random(), t1);
return new TokenStreamComponents(t, t2);
@@ -384,8 +376,8 @@
// seed:
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- final Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ final Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false);
final TokenStream t1 = new MockHoleInjectingTokenFilter(random(), t);
final TokenStream t2 = new MockGraphTokenFilter(random(), t1);
return new TokenStreamComponents(t, t2);
Index: lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestLookaheadTokenFilter.java (working copy)
@@ -26,9 +26,9 @@
public void testRandomStrings() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
Random random = random();
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, random.nextBoolean());
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, random.nextBoolean());
TokenStream output = new MockRandomLookaheadTokenFilter(random, tokenizer);
return new TokenStreamComponents(tokenizer, output);
}
@@ -55,8 +55,8 @@
public void testNeverCallingPeek() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, random().nextBoolean());
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, random().nextBoolean());
TokenStream output = new NeverPeeksLookaheadTokenFilter(tokenizer);
return new TokenStreamComponents(tokenizer, output);
}
@@ -67,9 +67,8 @@
public void testMissedFirstToken() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- Tokenizer source = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer source = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TrivialLookaheadFilter filter = new TrivialLookaheadFilter(source);
return new TokenStreamComponents(source, filter);
}
Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy)
@@ -187,8 +187,8 @@
public void testTooLongToken() throws Exception {
Analyzer whitespace = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false, 5);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(MockTokenizer.WHITESPACE, false, 5);
return new TokenStreamComponents(t, t);
}
};
@@ -235,8 +235,8 @@
final int limit = _TestUtil.nextInt(random(), 0, 500);
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(reader, dfa, lowercase, limit);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(dfa, lowercase, limit);
return new TokenStreamComponents(t, t);
}
};
Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java (working copy)
@@ -26,8 +26,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: lucene/core/src/test/org/apache/lucene/analysis/TestToken.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestToken.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestToken.java (working copy)
@@ -224,7 +224,8 @@
}
public void testTokenAttributeFactory() throws Exception {
- TokenStream ts = new MockTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo bar"), MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ TokenStream ts = new MockTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ ((Tokenizer)ts).setReader(new StringReader("foo bar"));
assertTrue("SenselessAttribute is not implemented by SenselessAttributeImpl",
ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
Index: lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/codecs/lucene41/TestBlockPostingsFormat3.java (working copy)
@@ -69,8 +69,8 @@
Directory dir = newDirectory();
Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
if (fieldName.contains("payloadsFixed")) {
TokenFilter filter = new MockFixedLengthPayloadFilter(new Random(0), tokenizer, 1);
return new TokenStreamComponents(tokenizer, filter);
Index: lucene/core/src/test/org/apache/lucene/document/TestDocument.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/document/TestDocument.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/document/TestDocument.java (working copy)
@@ -17,10 +17,12 @@
* limitations under the License.
*/
+import java.io.IOException;
import java.io.StringReader;
import java.util.List;
import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
@@ -314,10 +316,14 @@
// LUCENE-3616
public void testInvalidFields() {
try {
- new Field("foo", new MockTokenizer(new StringReader("")), StringField.TYPE_STORED);
+ Tokenizer tok = new MockTokenizer();
+ tok.setReader(new StringReader(""));
+ new Field("foo", tok, StringField.TYPE_STORED);
fail("did not hit expected exc");
} catch (IllegalArgumentException iae) {
// expected
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
}
}
Index: lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy)
@@ -106,8 +106,8 @@
public void testPositionIncrementGap() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
}
@Override
@@ -141,8 +141,8 @@
public void testTokenReuse() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, new TokenFilter(tokenizer) {
boolean first = true;
AttributeSource.State state;
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy)
@@ -1255,26 +1255,38 @@
Field f = new Field("binary", b, 10, 17, customType);
customType.setIndexed(true);
- f.setTokenStream(new MockTokenizer(new StringReader("doc1field1"), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer doc1field1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc1field1.setReader(new StringReader("doc1field1"));
+ f.setTokenStream(doc1field1);
FieldType customType2 = new FieldType(TextField.TYPE_STORED);
Field f2 = newField("string", "value", customType2);
- f2.setTokenStream(new MockTokenizer(new StringReader("doc1field2"), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer doc1field2 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc1field2.setReader(new StringReader("doc1field2"));
+ f2.setTokenStream(doc1field2);
doc.add(f);
doc.add(f2);
w.addDocument(doc);
// add 2 docs to test in-memory merging
- f.setTokenStream(new MockTokenizer(new StringReader("doc2field1"), MockTokenizer.WHITESPACE, false));
- f2.setTokenStream(new MockTokenizer(new StringReader("doc2field2"), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer doc2field1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc2field1.setReader(new StringReader("doc2field1"));
+ f.setTokenStream(doc2field1);
+ final MockTokenizer doc2field2 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc2field2.setReader(new StringReader("doc2field2"));
+ f2.setTokenStream(doc2field2);
w.addDocument(doc);
// force segment flush so we can force a segment merge with doc3 later.
w.commit();
- f.setTokenStream(new MockTokenizer(new StringReader("doc3field1"), MockTokenizer.WHITESPACE, false));
- f2.setTokenStream(new MockTokenizer(new StringReader("doc3field2"), MockTokenizer.WHITESPACE, false));
+ final MockTokenizer doc3field1 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc3field1.setReader(new StringReader("doc3field1"));
+ f.setTokenStream(doc3field1);
+ final MockTokenizer doc3field2 = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ doc3field2.setReader(new StringReader("doc3field2"));
+ f2.setTokenStream(doc3field2);
w.addDocument(doc);
w.commit();
@@ -1592,8 +1604,8 @@
static final class StringSplitAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new StringSplitTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new StringSplitTokenizer());
}
}
@@ -1602,13 +1614,8 @@
private int upto;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- public StringSplitTokenizer(Reader r) {
- super(r);
- try {
- setReader(r);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ public StringSplitTokenizer() {
+ super();
}
@Override
@@ -1875,7 +1882,7 @@
public void testDontInvokeAnalyzerForUnAnalyzedFields() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected TokenStreamComponents createComponents(String fieldName) {
throw new IllegalStateException("don't invoke me!");
}
@@ -1936,8 +1943,8 @@
Directory dir = newDirectory();
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
return new TokenStreamComponents(tokenizer, stream);
}
@@ -1966,8 +1973,8 @@
final Automaton secondSet = BasicAutomata.makeString("foobar");
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
TokenStream stream = new MockTokenFilter(tokenizer, MockTokenFilter.ENGLISH_STOPSET);
stream = new MockTokenFilter(stream, new CharacterRunAutomaton(secondSet));
return new TokenStreamComponents(tokenizer, stream);
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (working copy)
@@ -187,8 +187,8 @@
// no payloads
analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
}
};
} else {
@@ -196,8 +196,8 @@
final int length = random().nextInt(200);
analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockFixedLengthPayloadFilter(random(), tokenizer, length));
}
};
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy)
@@ -979,8 +979,8 @@
// note this test explicitly disables payloads
final Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
}
};
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setRAMBufferSizeMB(1.0).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMaxBufferedDeleteTerms(IndexWriterConfig.DISABLE_AUTO_FLUSH));
Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy)
@@ -390,8 +390,8 @@
Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
@@ -456,8 +456,8 @@
IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
return new TokenStreamComponents(tokenizer, new TokenFilter(tokenizer) {
private int count = 0;
@@ -593,8 +593,8 @@
public void testDocumentsWriterExceptions() throws IOException {
Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
@@ -688,8 +688,8 @@
public void testDocumentsWriterExceptionThreads() throws Exception {
Analyzer analyzer = new Analyzer(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
return new TokenStreamComponents(tokenizer, new CrashingFilter(fieldName, tokenizer));
}
@@ -1365,7 +1365,8 @@
if (docCount == 4) {
Field f = newTextField("crash", "", Field.Store.NO);
doc.add(f);
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader("crash me on the 4th token"));
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
f.setTokenStream(new CrashingFilter("crash", tokenizer));
}
@@ -1444,7 +1445,8 @@
if (docCount == crashAt) {
Field f = newTextField("crash", "", Field.Store.NO);
doc.add(f);
- MockTokenizer tokenizer = new MockTokenizer(new StringReader("crash me on the 4th token"), MockTokenizer.WHITESPACE, false);
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ tokenizer.setReader(new StringReader("crash me on the 4th token"));
tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases.
f.setTokenStream(new CrashingFilter("crash", tokenizer));
}
Index: lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (working copy)
@@ -70,8 +70,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
}
};
Directory directory = new SeekCountingDirectory(new RAMDirectory());
Index: lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy)
@@ -112,8 +112,8 @@
private static class PayloadAnalyzer extends Analyzer {
private final AtomicInteger payloadCount = new AtomicInteger(-1);
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new PayloadFilter(payloadCount, tokenizer));
}
Index: lucene/core/src/test/org/apache/lucene/index/TestPayloads.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (working copy)
@@ -376,9 +376,9 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
PayloadData payload = fieldToData.get(fieldName);
- Tokenizer ts = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ Tokenizer ts = new MockTokenizer(MockTokenizer.WHITESPACE, false);
TokenStream tokenStream = (payload != null) ?
new PayloadFilter(ts, payload.data, payload.offset, payload.length) : ts;
return new TokenStreamComponents(ts, tokenStream);
@@ -588,7 +588,8 @@
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
Field field = new TextField("field", "", Field.Store.NO);
- TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("here we go"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
@@ -599,7 +600,8 @@
assertTrue(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
writer.addDocument(doc);
- ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+ ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("another"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
writer.addDocument(doc);
@@ -620,7 +622,8 @@
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
Document doc = new Document();
Field field = new TextField("field", "", Field.Store.NO);
- TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("here we go"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
@@ -632,7 +635,9 @@
field2.setTokenStream(ts);
doc.add(field2);
Field field3 = new TextField("field", "", Field.Store.NO);
- ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+ ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("nopayload"));
+
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field3.setTokenStream(ts);
doc.add(field3);
Index: lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (working copy)
@@ -24,6 +24,7 @@
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -48,7 +49,8 @@
customType.setStoreTermVectorPayloads(true);
customType.setStoreTermVectorOffsets(random().nextBoolean());
Field field = new Field("field", "", customType);
- TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("here we go"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
@@ -61,7 +63,8 @@
field.setTokenStream(ts);
writer.addDocument(doc);
- ts = new MockTokenizer(new StringReader("another"), MockTokenizer.WHITESPACE, true);
+ ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("another"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
writer.addDocument(doc);
@@ -91,7 +94,8 @@
customType.setStoreTermVectorPayloads(true);
customType.setStoreTermVectorOffsets(random().nextBoolean());
Field field = new Field("field", "", customType);
- TokenStream ts = new MockTokenizer(new StringReader("here we go"), MockTokenizer.WHITESPACE, true);
+ TokenStream ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("here we go"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field.setTokenStream(ts);
doc.add(field);
@@ -103,7 +107,8 @@
field2.setTokenStream(ts);
doc.add(field2);
Field field3 = new Field("field", "", customType);
- ts = new MockTokenizer(new StringReader("nopayload"), MockTokenizer.WHITESPACE, true);
+ ts = new MockTokenizer(MockTokenizer.WHITESPACE, true);
+ ((Tokenizer)ts).setReader(new StringReader("nopayload"));
assertFalse(ts.hasAttribute(PayloadAttribute.class));
field3.setTokenStream(ts);
doc.add(field3);
Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (working copy)
@@ -143,8 +143,8 @@
private final PositionIncrementAttribute posIncrAtt;
private final OffsetAttribute offsetAtt;
- public MyTokenizer(Reader reader) {
- super(reader);
+ public MyTokenizer() {
+ super();
termAtt = addAttribute(CharTermAttribute.class);
posIncrAtt = addAttribute(PositionIncrementAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
@@ -177,8 +177,8 @@
private class MyAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MyTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MyTokenizer());
}
}
Index: lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java (working copy)
@@ -42,8 +42,8 @@
CharTermAttribute termAtt;
String value;
- public RepeatingTokenizer(Reader reader, String val, Random random, float percentDocs, int maxTF) {
- super(reader);
+ public RepeatingTokenizer(String val, Random random, float percentDocs, int maxTF) {
+ super();
this.value = val;
this.random = random;
this.percentDocs = percentDocs;
@@ -80,8 +80,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new RepeatingTokenizer(reader, val, random, percentDocs, maxTF));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new RepeatingTokenizer(val, random, percentDocs, maxTF));
}
};
Index: lucene/core/src/test/org/apache/lucene/search/FuzzyTermOnShortTermsTest.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/FuzzyTermOnShortTermsTest.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/FuzzyTermOnShortTermsTest.java (working copy)
@@ -75,8 +75,8 @@
public static Analyzer getAnalyzer(){
return new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
Index: lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (working copy)
@@ -53,8 +53,8 @@
directory = newDirectory();
Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
}
@Override
Index: lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (working copy)
@@ -58,8 +58,8 @@
public void testSetPosition() throws Exception {
Analyzer analyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new Tokenizer(reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new Tokenizer() {
// TODO: use CannedTokenStream
private final String[] TOKENS = {"1", "2", "3", "4", "5"};
private final int[] INCREMENTS = {1, 2, 1, 0, 1};
Index: lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/TestTermRangeQuery.java (working copy)
@@ -206,8 +206,8 @@
boolean done = false;
CharTermAttribute termAtt;
- public SingleCharTokenizer(Reader r) {
- super(r);
+ public SingleCharTokenizer() {
+ super();
termAtt = addAttribute(CharTermAttribute.class);
}
@@ -234,8 +234,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new SingleCharTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SingleCharTokenizer());
}
}
Index: lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (working copy)
@@ -63,8 +63,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Index: lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy)
@@ -55,8 +55,8 @@
private static class PayloadAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Index: lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (working copy)
@@ -68,8 +68,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(result, new PayloadFilter(result, fieldName));
}
}
Index: lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/spans/TestBasics.java (working copy)
@@ -102,8 +102,8 @@
public static void beforeClass() throws Exception {
simplePayloadAnalyzer = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new SimplePayloadFilter(tokenizer));
}
};
Index: lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (working copy)
@@ -472,8 +472,8 @@
final class PayloadAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(result, new PayloadFilter(result));
}
}
@@ -526,8 +526,8 @@
public final class TestPayloadAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(result, new PayloadFilter(result));
}
}
Index: lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java (revision 1556706)
+++ lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java (working copy)
@@ -119,8 +119,8 @@
/** adds synonym of "dog" for "dogs". */
static class MockSynonymAnalyzer extends Analyzer {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer();
return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
}
}
@@ -180,8 +180,8 @@
protected static class SimpleCJKTokenizer extends Tokenizer {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- public SimpleCJKTokenizer(Reader input) {
- super(input);
+ public SimpleCJKTokenizer() {
+ super();
}
@Override
@@ -197,8 +197,8 @@
private class SimpleCJKAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new SimpleCJKTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SimpleCJKTokenizer());
}
}
@@ -272,8 +272,8 @@
static class MockCJKSynonymAnalyzer extends Analyzer {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new SimpleCJKTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new SimpleCJKTokenizer();
return new TokenStreamComponents(tokenizer, new MockCJKSynonymFilter(tokenizer));
}
}
Index: lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
===================================================================
--- lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 1556706)
+++ lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy)
@@ -1954,8 +1954,8 @@
* java.io.Reader)
*/
@Override
- public TokenStreamComponents createComponents(String arg0, Reader arg1) {
- Tokenizer stream = new MockTokenizer(arg1, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String arg0) {
+ Tokenizer stream = new MockTokenizer(MockTokenizer.SIMPLE, true);
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
Index: lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java
===================================================================
--- lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (revision 1556706)
+++ lucene/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (working copy)
@@ -27,23 +27,21 @@
public void testFilter() throws Exception {
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
- MockTokenizer stream = new MockTokenizer(new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"),
+ MockTokenizer stream = new MockTokenizer(
MockTokenizer.WHITESPACE, false);
+ stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
stream.setEnableChecks(false);
OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
- stream = new MockTokenizer(new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"),
- MockTokenizer.WHITESPACE, false);
+ stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 12);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
- stream = new MockTokenizer(new StringReader(
- "short toolong evenmuchlongertext a ab toolong foo"),
- MockTokenizer.WHITESPACE, false);
+ stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ stream.setReader(new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 30);
assertTokenStreamContents(filter, new String[] {"short", "toolong",
@@ -52,8 +50,8 @@
checkOneTerm(new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false);
return new TokenStreamComponents(tokenizer, new OffsetLimitTokenFilter(tokenizer, 10));
}
Index: lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
===================================================================
--- lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (revision 1556706)
+++ lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java (working copy)
@@ -198,8 +198,8 @@
static final class BigramAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new BasicNGramTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new BasicNGramTokenizer());
}
}
@@ -221,20 +221,20 @@
private int charBufferIndex;
private int charBufferLen;
- public BasicNGramTokenizer( Reader in ){
- this( in, DEFAULT_N_SIZE );
+ public BasicNGramTokenizer( ){
+ this( DEFAULT_N_SIZE );
}
- public BasicNGramTokenizer( Reader in, int n ){
- this( in, n, DEFAULT_DELIMITERS );
+ public BasicNGramTokenizer( int n ){
+ this( n, DEFAULT_DELIMITERS );
}
- public BasicNGramTokenizer( Reader in, String delimiters ){
- this( in, DEFAULT_N_SIZE, delimiters );
+ public BasicNGramTokenizer( String delimiters ){
+ this( DEFAULT_N_SIZE, delimiters );
}
- public BasicNGramTokenizer( Reader in, int n, String delimiters ){
- super(in);
+ public BasicNGramTokenizer(int n, String delimiters ){
+ super();
this.n = n;
this.delimiters = delimiters;
startTerm = 0;
Index: lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java
===================================================================
--- lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (revision 1556706)
+++ lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (working copy)
@@ -297,8 +297,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) {
final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
int p = 0;
Index: lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
===================================================================
--- lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 1556706)
+++ lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy)
@@ -261,8 +261,8 @@
case 1: return new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
case 2: return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer();
return new TokenStreamComponents(tokenizer, new CrazyTokenFilter(tokenizer));
}
};
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/analyzing/TestAnalyzingQueryParser.java (working copy)
@@ -253,8 +253,8 @@
final static class ASCIIAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new FoldingFilter(result));
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java (working copy)
@@ -125,8 +125,8 @@
private class MultiAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new TestFilter(result));
}
}
@@ -196,8 +196,8 @@
private class PosIncrementAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new TestPosIncrementFilter(result));
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (working copy)
@@ -327,8 +327,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return stdAnalyzer.createComponents(fieldName, reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ return stdAnalyzer.createComponents(fieldName);
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java (working copy)
@@ -48,8 +48,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new CannedTokenizer(reader, tokens));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new CannedTokenizer(tokens));
}
}
@@ -60,8 +60,8 @@
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- public CannedTokenizer(Reader reader, TokenAndPos[] tokens) {
- super(reader);
+ public CannedTokenizer(TokenAndPos[] tokens) {
+ super();
this.tokens = tokens;
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java (working copy)
@@ -153,6 +153,7 @@
//
// This test is here as a safety, in case that ant step
// doesn't work for some reason.
+ @SuppressWarnings("rawtype")
public void testProtectedCtors() throws Exception {
try {
QueryParser.class.getConstructor(new Class[] {CharStream.class});
@@ -321,8 +322,8 @@
/** adds synonym of "dog" for "dogs". */
static class MockSynonymAnalyzer extends Analyzer {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer();
return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
}
}
@@ -391,8 +392,8 @@
static class MockCJKSynonymAnalyzer extends Analyzer {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new SimpleCJKTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new SimpleCJKTokenizer();
return new TokenStreamComponents(tokenizer, new MockCJKSynonymFilter(tokenizer));
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/precedence/TestPrecedenceQueryParser.java (working copy)
@@ -130,8 +130,8 @@
/** Filters MockTokenizer with StopFilter. */
@Override
- public final TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public final TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer( MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiAnalyzerQPHelper.java (working copy)
@@ -146,8 +146,8 @@
private class MultiAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new TestFilter(result));
}
}
@@ -213,8 +213,8 @@
private class PosIncrementAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new TestPosIncrementFilter(result));
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (working copy)
@@ -363,8 +363,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return stdAnalyzer.createComponents(fieldName, reader);
+ public TokenStreamComponents createComponents(String fieldName) {
+ return stdAnalyzer.createComponents(fieldName);
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (working copy)
@@ -147,8 +147,8 @@
/** Filters MockTokenizer with StopFilter. */
@Override
- public final TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public final TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
}
}
@@ -345,8 +345,8 @@
private class SimpleCJKTokenizer extends Tokenizer {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- public SimpleCJKTokenizer(Reader input) {
- super(input);
+ public SimpleCJKTokenizer() {
+ super();
}
@Override
@@ -362,8 +362,8 @@
private class SimpleCJKAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new SimpleCJKTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SimpleCJKTokenizer());
}
}
@@ -1267,8 +1267,8 @@
private final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class);
private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
- public CannedTokenizer(Reader reader) {
- super(reader);
+ public CannedTokenizer() {
+ super();
}
@Override
@@ -1303,8 +1303,8 @@
private class CannedAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String ignored, Reader alsoIgnored) {
- return new TokenStreamComponents(new CannedTokenizer(alsoIgnored));
+ public TokenStreamComponents createComponents(String ignored) {
+ return new TokenStreamComponents(new CannedTokenizer());
}
}
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestStandardQP.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestStandardQP.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestStandardQP.java (working copy)
@@ -118,10 +118,8 @@
// TODO implement LUCENE-2566 and remove this (override)method
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName,
- Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader,
- MockTokenizer.WHITESPACE, false));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
}
};
assertQueryEquals("a - b", a, "a -b");
Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java
===================================================================
--- lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (revision 1556706)
+++ lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (working copy)
@@ -118,8 +118,8 @@
/** Filters MockTokenizer with StopFilter. */
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
}
}
@@ -250,8 +250,8 @@
protected static class SimpleCJKTokenizer extends Tokenizer {
private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- public SimpleCJKTokenizer(Reader input) {
- super(input);
+ public SimpleCJKTokenizer() {
+ super();
}
@Override
@@ -267,8 +267,8 @@
private class SimpleCJKAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new SimpleCJKTokenizer(reader));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new SimpleCJKTokenizer());
}
}
@@ -403,8 +403,8 @@
// +,-,! should be directly adjacent to operand (i.e. not separated by whitespace) to be treated as an operator
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
}
};
assertQueryEquals("a - b", a, "a - b");
@@ -1133,8 +1133,8 @@
super();
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer( MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
}
}
@@ -1145,8 +1145,8 @@
super();
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true));
+ public TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
}
}
@@ -1176,8 +1176,8 @@
}
private class MockCollationAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(tokenizer, new MockCollationFilter(tokenizer));
}
}
Index: lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
===================================================================
--- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (revision 1556706)
+++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (working copy)
@@ -397,8 +397,8 @@
final CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "a");
Analyzer indexAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokens = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokens = new MockTokenizer();
return new TokenStreamComponents(tokens,
new StopFilter(TEST_VERSION_CURRENT, tokens, stopWords));
}
@@ -406,8 +406,8 @@
Analyzer queryAnalyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokens = new MockTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokens = new MockTokenizer();
return new TokenStreamComponents(tokens,
new SuggestStopFilter(tokens, stopWords));
}
Index: lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
===================================================================
--- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (revision 1556706)
+++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (working copy)
@@ -273,8 +273,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -346,8 +346,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -424,8 +424,8 @@
private final Analyzer getUnusualAnalyzer() {
return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -631,8 +631,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(factory, reader, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(factory, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
tokenizer.setEnableChecks(true);
TokenStream next;
if (numStopChars != 0) {
@@ -948,8 +948,8 @@
public void testDupSurfaceFormsMissingResults() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -1007,8 +1007,8 @@
public void testDupSurfaceFormsMissingResults2() throws Exception {
Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -1077,8 +1077,8 @@
public void test0ByteKeys() throws Exception {
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -1147,8 +1147,8 @@
final Analyzer a = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@Override
Index: lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java
===================================================================
--- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (revision 1556706)
+++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/FuzzySuggesterTest.java (working copy)
@@ -227,8 +227,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -308,8 +308,8 @@
final Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
int tokenStreamCounter = 0;
@@ -382,8 +382,8 @@
private final Analyzer getUnusualAnalyzer() {
return new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
return new TokenStreamComponents(tokenizer) {
@@ -579,8 +579,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
tokenizer.setEnableChecks(true);
TokenStream next;
if (numStopChars != 0) {
Index: lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
===================================================================
--- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (revision 1556706)
+++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (working copy)
@@ -231,8 +231,8 @@
// Just deletes "of"
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer();
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "of");
return new TokenStreamComponents(tokenizer, new StopFilter(TEST_VERSION_CURRENT, tokenizer, stopSet));
}
@@ -259,8 +259,8 @@
// Just deletes "of"
Analyzer a = new Analyzer() {
@Override
- public TokenStreamComponents createComponents(String field, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader);
+ public TokenStreamComponents createComponents(String field) {
+ Tokenizer tokenizer = new MockTokenizer();
CharArraySet stopSet = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "of");
return new TokenStreamComponents(tokenizer, new StopFilter(TEST_VERSION_CURRENT, tokenizer, stopSet));
}
Index: lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java
===================================================================
--- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java (revision 1556706)
+++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestSuggestStopFilter.java (working copy)
@@ -22,6 +22,7 @@
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.util.CharArraySet;
@@ -29,7 +30,8 @@
public void testEndNotStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
assertTokenStreamContents(filter,
new String[] {"go", "to"},
@@ -46,9 +48,9 @@
public void testEndIsStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to "));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to "));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
-
filter = new SuggestStopFilter(stream, stopWords);
assertTokenStreamContents(filter,
new String[] {"go"},
@@ -65,7 +67,8 @@
public void testMidStopWord() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to");
- TokenStream stream = new MockTokenizer(new StringReader("go to school"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to school"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -84,7 +87,8 @@
public void testMultipleStopWords() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the school"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the school"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -103,7 +107,8 @@
public void testMultipleStopWordsEnd() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the"));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the"));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
@@ -122,7 +127,8 @@
public void testMultipleStopWordsEnd2() throws Exception {
CharArraySet stopWords = StopFilter.makeStopSet(TEST_VERSION_CURRENT, "to", "the", "a");
- TokenStream stream = new MockTokenizer(new StringReader("go to a the "));
+ Tokenizer stream = new MockTokenizer();
+ stream.setReader(new StringReader("go to a the "));
TokenStream filter = new SuggestStopFilter(stream, stopWords);
filter = new SuggestStopFilter(stream, stopWords);
Index: lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1556706)
+++ lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy)
@@ -972,4 +972,29 @@
}
return ret;
}
+
+ protected static MockTokenizer whitespaceMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer whitespaceMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer keywordMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer keywordMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.KEYWORD, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
}
Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (revision 1556706)
+++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockAnalyzer.java (working copy)
@@ -87,8 +87,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase, maxTokenLength);
+ public TokenStreamComponents createComponents(String fieldName) {
+ MockTokenizer tokenizer = new MockTokenizer(runAutomaton, lowerCase, maxTokenLength);
tokenizer.setEnableChecks(enableChecks);
MockTokenFilter filt = new MockTokenFilter(tokenizer, filter);
return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName));
Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java (revision 1556706)
+++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockBytesAnalyzer.java (working copy)
@@ -17,8 +17,6 @@
* limitations under the License.
*/
-import java.io.Reader;
-
/**
* Analyzer for testing that encodes terms as UTF-16 bytes.
*/
@@ -26,8 +24,8 @@
private final MockBytesAttributeFactory factory = new MockBytesAttributeFactory();
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer t = new MockTokenizer(factory, reader, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer t = new MockTokenizer(factory, MockTokenizer.KEYWORD, false, MockTokenizer.DEFAULT_MAX_TOKEN_LENGTH);
return new TokenStreamComponents(t);
}
}
Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision 1556706)
+++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java (working copy)
@@ -34,8 +34,8 @@
public final class MockPayloadAnalyzer extends Analyzer {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer result = new MockTokenizer( MockTokenizer.WHITESPACE, true);
return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName));
}
}
Index: lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
===================================================================
--- lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (revision 1556706)
+++ lucene/test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java (working copy)
@@ -89,35 +89,34 @@
// evil: but we don't change the behavior with this random, we only switch up how we read
private final Random random = new Random(RandomizedContext.current().getRandom().nextLong());
- public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
- super(factory, input);
+ public MockTokenizer(AttributeFactory factory, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
+ super(factory);
this.runAutomaton = runAutomaton;
this.lowerCase = lowerCase;
this.state = runAutomaton.getInitialState();
- this.streamState = State.SETREADER;
this.maxTokenLength = maxTokenLength;
}
- public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
- this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, runAutomaton, lowerCase, maxTokenLength);
+ public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, runAutomaton, lowerCase, maxTokenLength);
}
- public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
- this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ public MockTokenizer(CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
- /** Calls {@link #MockTokenizer(Reader, CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
- public MockTokenizer(Reader input) {
- this(input, WHITESPACE, true);
+ /** Calls {@link #MockTokenizer(CharacterRunAutomaton, boolean) MockTokenizer(Reader, WHITESPACE, true)} */
+ public MockTokenizer() {
+ this(WHITESPACE, true);
}
- public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
- this(factory, input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ public MockTokenizer(AttributeFactory factory, CharacterRunAutomaton runAutomaton, boolean lowerCase) {
+ this(factory, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
}
- /** Calls {@link #MockTokenizer(org.apache.lucene.util.AttributeSource.AttributeFactory,Reader,CharacterRunAutomaton,boolean)
+ /** Calls {@link #MockTokenizer(org.apache.lucene.util.AttributeSource.AttributeFactory,CharacterRunAutomaton,boolean)
* MockTokenizer(AttributeFactory, Reader, WHITESPACE, true)} */
- public MockTokenizer(AttributeFactory factory, Reader input) {
- this(input, WHITESPACE, true);
+ public MockTokenizer(AttributeFactory factory) {
+ this(factory, WHITESPACE, true);
}
@Override
Index: solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
===================================================================
--- solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (revision 1556706)
+++ solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java (working copy)
@@ -60,8 +60,8 @@
}
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
- Tokenizer tk = tokenizer.create( aReader );
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tk = tokenizer.create();
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
Index: solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
===================================================================
--- solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (revision 1556706)
+++ solr/core/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (working copy)
@@ -19,6 +19,7 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.*;
import org.apache.lucene.analysis.util.CharFilterFactory;
import org.apache.lucene.analysis.util.TokenFilterFactory;
@@ -110,7 +111,12 @@
}
}
- TokenStream tokenStream = tfac.create(tokenizerChain.initReader(null, new StringReader(value)));
+ TokenStream tokenStream = tfac.create();
+ try {
+ ((Tokenizer)tokenStream).setReader(tokenizerChain.initReader(null, new StringReader(value)));
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
List tokens = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
Index: solr/core/src/java/org/apache/solr/schema/BoolField.java
===================================================================
--- solr/core/src/java/org/apache/solr/schema/BoolField.java (revision 1556706)
+++ solr/core/src/java/org/apache/solr/schema/BoolField.java (working copy)
@@ -67,8 +67,8 @@
protected final static Analyzer boolAnalyzer = new SolrAnalyzer() {
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new Tokenizer(reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new Tokenizer() {
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
boolean done = false;
Index: solr/core/src/java/org/apache/solr/schema/FieldType.java
===================================================================
--- solr/core/src/java/org/apache/solr/schema/FieldType.java (revision 1556706)
+++ solr/core/src/java/org/apache/solr/schema/FieldType.java (working copy)
@@ -439,8 +439,8 @@
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer ts = new Tokenizer(reader) {
+ public TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer ts = new Tokenizer() {
final char[] cbuf = new char[maxChars];
final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
Index: solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java
===================================================================
--- solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (revision 1556706)
+++ solr/core/src/java/org/apache/solr/schema/PreAnalyzedField.java (working copy)
@@ -86,8 +86,8 @@
return new SolrAnalyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- return new TokenStreamComponents(new PreAnalyzedTokenizer(reader, parser));
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new PreAnalyzedTokenizer(parser));
}
};
@@ -198,7 +198,8 @@
if (val == null || val.trim().length() == 0) {
return null;
}
- PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(new StringReader(val), parser);
+ PreAnalyzedTokenizer parse = new PreAnalyzedTokenizer(parser);
+ parse.setReader(new StringReader(val));
parse.reset(); // consume
org.apache.lucene.document.FieldType type = createFieldType(field);
if (type == null) {
@@ -251,12 +252,8 @@
private String stringValue = null;
private byte[] binaryValue = null;
private PreAnalyzedParser parser;
- private Reader lastReader;
- private Reader input; // hides original input since we replay saved states (and dont reuse)
- public PreAnalyzedTokenizer(Reader reader, PreAnalyzedParser parser) {
- super(reader);
- this.input = reader;
+ public PreAnalyzedTokenizer(PreAnalyzedParser parser) {
this.parser = parser;
}
@@ -271,7 +268,7 @@
public byte[] getBinaryValue() {
return binaryValue;
}
-
+
@Override
public final boolean incrementToken() {
// lazy init the iterator
@@ -291,8 +288,8 @@
@Override
public final void reset() throws IOException {
// NOTE: this acts like rewind if you call it again
- if (input != lastReader) {
- lastReader = input;
+ if (it == null) {
+ super.reset();
cachedStates.clear();
stringValue = null;
binaryValue = null;
@@ -307,12 +304,6 @@
}
it = cachedStates.iterator();
}
-
- @Override
- public void close() throws IOException {
- super.close();
- lastReader = null; // just a ref, null for gc
- }
}
}
Index: solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java
===================================================================
--- solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (revision 1556706)
+++ solr/core/src/test/org/apache/solr/analysis/LegacyHTMLStripCharFilterTest.java (working copy)
@@ -260,8 +260,8 @@
Analyzer analyzer = new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
return new TokenStreamComponents(tokenizer, tokenizer);
}
Index: solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java
===================================================================
--- solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (revision 1556706)
+++ solr/core/src/test/org/apache/solr/analysis/TestReversedWildcardFilterFactory.java (working copy)
@@ -66,7 +66,7 @@
String text = "simple text";
args.put("withOriginal", "true");
ReversedWildcardFilterFactory factory = new ReversedWildcardFilterFactory(args);
- TokenStream input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
+ TokenStream input = factory.create(whitespaceMockTokenizer(text));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "simple", "\u0001txet", "text" },
new int[] { 1, 0, 1, 0 });
@@ -74,7 +74,7 @@
// now without original tokens
args.put("withOriginal", "false");
factory = new ReversedWildcardFilterFactory(args);
- input = factory.create(new MockTokenizer(new StringReader(text), MockTokenizer.WHITESPACE, false));
+ input = factory.create(whitespaceMockTokenizer(text));
assertTokenStreamContents(input,
new String[] { "\u0001elpmis", "\u0001txet" },
new int[] { 1, 1 });
Index: solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java
===================================================================
--- solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (revision 1556706)
+++ solr/core/src/test/org/apache/solr/analysis/TestWordDelimiterFilterFactory.java (working copy)
@@ -211,13 +211,11 @@
WordDelimiterFilterFactory factoryDefault = new WordDelimiterFilterFactory(args);
factoryDefault.inform(loader);
- TokenStream ts = factoryDefault.create(
- new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
+ TokenStream ts = factoryDefault.create(whitespaceMockTokenizer(testText));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "5", "400", "00", "540000", "at", "25", "interest", "rate", "interestrate" });
- ts = factoryDefault.create(
- new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
+ ts = factoryDefault.create(whitespaceMockTokenizer("foo\u200Dbar"));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo", "bar", "foobar" });
@@ -235,14 +233,12 @@
WordDelimiterFilterFactory factoryCustom = new WordDelimiterFilterFactory(args);
factoryCustom.inform(loader);
- ts = factoryCustom.create(
- new MockTokenizer(new StringReader(testText), MockTokenizer.WHITESPACE, false));
+ ts = factoryCustom.create(whitespaceMockTokenizer(testText));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "I", "borrowed", "$5,400.00", "at", "25%", "interest", "rate", "interestrate" });
/* test custom behavior with a char > 0x7F, because we had to make a larger byte[] */
- ts = factoryCustom.create(
- new MockTokenizer(new StringReader("foo\u200Dbar"), MockTokenizer.WHITESPACE, false));
+ ts = factoryCustom.create(whitespaceMockTokenizer("foo\u200Dbar"));
BaseTokenStreamTestCase.assertTokenStreamContents(ts,
new String[] { "foo\u200Dbar" });
}
Index: solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java
===================================================================
--- solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java (revision 1556706)
+++ solr/core/src/test/org/apache/solr/spelling/TestSuggestSpellingConverter.java (working copy)
@@ -48,8 +48,8 @@
// lowercases, removes field names, other syntax, collapses runs of whitespace, etc.
converter.setAnalyzer(new Analyzer() {
@Override
- protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- Tokenizer tokenizer = new KeywordTokenizer(reader);
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
TokenStream filter = new PatternReplaceFilter(tokenizer,
Pattern.compile("([^\\p{L}\\p{M}\\p{N}\\p{Cs}]*[\\p{L}\\p{M}\\p{N}\\p{Cs}\\_]+:)|([^\\p{L}\\p{M}\\p{N}\\p{Cs}])+"), " ", true);
filter = new LowerCaseFilter(TEST_VERSION_CURRENT, filter);
Index: solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
===================================================================
--- solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (revision 1556706)
+++ solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java (working copy)
@@ -21,6 +21,8 @@
import java.io.File;
import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
import java.util.Arrays;
@@ -41,6 +43,7 @@
import org.apache.commons.codec.Charsets;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@@ -174,6 +177,19 @@
}
}
+
+ protected static MockTokenizer whitespaceMockTokenizer(Reader input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(input);
+ return mockTokenizer;
+ }
+
+ protected static MockTokenizer whitespaceMockTokenizer(String input) throws IOException {
+ MockTokenizer mockTokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ mockTokenizer.setReader(new StringReader(input));
+ return mockTokenizer;
+ }
+
/**
* Call this from @BeforeClass to set up the test harness and update handler with no cores.
*
@@ -1770,4 +1786,5 @@
}
}
+
}
Index: solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java
===================================================================
--- solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (revision 1556706)
+++ solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (working copy)
@@ -52,8 +52,8 @@
}
@Override
- public MockTokenizer create(AttributeFactory factory, Reader input) {
- MockTokenizer t = new MockTokenizer(factory, input, pattern, false);
+ public MockTokenizer create(AttributeFactory factory) {
+ MockTokenizer t = new MockTokenizer(factory, pattern, false);
t.setEnableChecks(enableChecks);
return t;
}