Index: modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (revision 1239491) +++ modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestTypeTokenFilter.java (working copy) @@ -27,6 +27,7 @@ import java.io.IOException; import java.io.StringReader; +import java.util.Collections; import java.util.Set; @@ -83,6 +84,13 @@ stpf.close(); } + public void testTypeFilterWhitelist() throws IOException { + StringReader reader = new StringReader("121 is palindrome, while 123 is not"); + Set stopTypes = Collections.singleton(""); + TokenStream stream = new TypeTokenFilter(true, new StandardTokenizer(TEST_VERSION_CURRENT, reader), stopTypes, true); + assertTokenStreamContents(stream, new String[]{"121", "123"}); + } + // print debug info depending on VERBOSE private static void log(String s) { if (VERBOSE) { Index: modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java (revision 1239491) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/core/TypeTokenFilter.java (working copy) @@ -31,17 +31,24 @@ private final Set stopTypes; private final TypeAttribute typeAttribute = addAttribute(TypeAttribute.class); + private final boolean useWhiteList; - public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set stopTypes) { + public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set stopTypes, boolean useWhiteList) { super(enablePositionIncrements, input); this.stopTypes = stopTypes; + this.useWhiteList = useWhiteList; } + public TypeTokenFilter(boolean enablePositionIncrements, TokenStream input, Set stopTypes) { + this(enablePositionIncrements, input, stopTypes, false); + } + /** - * Returns the next input Token whose typeAttribute.type() is not a stop type. + * By default accept the token if its type is not a stop type. + * When the useWhiteList parameter is set to true then accept the token if its type is contained in the stopTypes */ @Override protected boolean accept() throws IOException { - return !stopTypes.contains(typeAttribute.type()); + return useWhiteList == stopTypes.contains(typeAttribute.type()); } }