Index: src/java/org/apache/lucene/analysis/StopFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/StopFilter.java (revision 544097) +++ src/java/org/apache/lucene/analysis/StopFilter.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.HashSet; +import java.util.LinkedList; import java.util.Set; /** @@ -29,6 +30,7 @@ private final Set stopWords; private final boolean ignoreCase; + private final boolean removeStopwordCollocates; /** * Construct a token stream filtering the given input. @@ -43,26 +45,51 @@ * TokenStream that are named in the array of words. */ public StopFilter(TokenStream in, String[] stopWords, boolean ignoreCase) { - super(in); - this.ignoreCase = ignoreCase; - this.stopWords = makeStopSet(stopWords, ignoreCase); + this(in, stopWords, false, false ); } - /** - * Construct a token stream filtering the given input. - * @param input - * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased - * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words - */ - public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase) - { - super(input); - this.ignoreCase = ignoreCase; - this.stopWords = stopWords; - } + /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + */ + public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase) + { + this(input, stopWords, false, false ); + } /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + * @param removeStopwordCollocates -When stopword is included in token at the same position, everything is removed. + */ + public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase, boolean removeStopwordCollocates) + { + super(input); + this.ignoreCase = ignoreCase; + this.stopWords = stopWords; + this.removeStopwordCollocates = removeStopwordCollocates ; + } + + /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + * @param removeStopwordCollocates -When stopword is included in token at the same position, everything is removed. + */ + public StopFilter(TokenStream in, String[] stopWords, boolean ignoreCase, boolean removeStopwordCollocates) { + super(in); + this.ignoreCase = ignoreCase; + this.stopWords = makeStopSet(stopWords, ignoreCase); + this.removeStopwordCollocates = removeStopwordCollocates ; + } + + /** * Constructs a filter which removes words from the input * TokenStream that are named in the Set. * It is crucial that an efficient Set implementation is used @@ -88,7 +115,7 @@ /** * - * @param stopWords + * @param stopWords * @param ignoreCase If true, all words are lower cased first. * @return a Set containing the words */ @@ -99,18 +126,81 @@ return stopTable; } + private final LinkedList inQueue = new LinkedList(); + private final LinkedList outQueue = new LinkedList(); + + private final Token removeStopwordCollocatesNext() throws IOException { + while (true) { + if (!outQueue.isEmpty()) { + return outQueue.removeFirst(); + } + Token token = null; + if (inQueue.isEmpty()) { + token = input.next(); + } else { + token = inQueue.removeFirst(); + } + if (null == token) { + return null; + } + + Token next = null; + if (inQueue.isEmpty()) { + next = input.next(); + } else { + next = inQueue.removeFirst(); + } + boolean stopword = false; + + LinkedList checkQueue = new LinkedList(); + checkQueue.addLast(token); + + String termText = ignoreCase ? token.termText.toLowerCase() + : token.termText; + if (stopWords.contains(termText)) { + stopword = true; + } + + while (next != null && next.getPositionIncrement() == 0) { + checkQueue.addLast(next); + termText = ignoreCase ? next.termText.toLowerCase() + : next.termText; + if (stopWords.contains(termText)) { + stopword = true; + } + if (inQueue.isEmpty()) { + next = input.next(); + } else { + next = inQueue.removeFirst(); + } + } + if (!stopword) { + for (Token out : checkQueue) { + outQueue.addLast(out); + } + } + if (next != null) { + inQueue.addFirst(next); + } + } + } /** * Returns the next input Token whose termText() is not a stop word. */ public final Token next() throws IOException { - // return the first non-stop word found - for (Token token = input.next(); token != null; token = input.next()) - { - String termText = ignoreCase ? token.termText.toLowerCase() : token.termText; - if (!stopWords.contains(termText)) - return token; - } - // reached EOS -- return null - return null; + if (removeStopwordCollocates) { + return removeStopwordCollocatesNext() ; + } else { + // return the first non-stop word found + for (Token token = input.next(); token != null; token = input.next()) + { + String termText = ignoreCase ? token.termText.toLowerCase() : token.termText; + if (!stopWords.contains(termText)) + return token; + } + // reached EOS -- return null + return null; + } } + } Index: src/test/org/apache/lucene/analysis/TestStopFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/TestStopFilter.java (revision 544097) +++ src/test/org/apache/lucene/analysis/TestStopFilter.java (working copy) @@ -18,6 +18,8 @@ import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; +import java.util.Iterator; import junit.framework.TestCase; @@ -44,5 +46,114 @@ assertEquals("Now", stream.next().termText()); assertEquals(null,stream.next()); } + + + public Token newToken(String termText, int pos) { + Token token = new Token(termText, 0, 0); + token.setPositionIncrement(pos); + return token; + } + public void testPosInc( String[] stopWords, boolean ignoreCase, final String expectation, final Token[] tokens) + throws IOException { + + final Iterator token = Arrays.asList(tokens).iterator(); + + final TokenStream ts = new StopFilter(new TokenStream() { + public Token next() { + return token.hasNext() ? token.next() : null; + } + }, stopWords, ignoreCase, true); + + final String result = tokenStream2String(ts); + assertEquals(expectation + " != " + result, expectation, result); +// System.out.println(result) ; + } + + + public static String tokenStream2String(TokenStream input) + throws IOException { + StringBuffer output = new StringBuffer(); + Token token = input.next(); + if (null != token) { + output.append(token.termText()); + } + for (token = input.next(); null != token; token = input.next()) { + output.append(" ").append(token.termText()); + } + input.close(); + return output.toString(); + } + + public void testPosIncs() throws IOException { + + String[] stopWords = new String[] { "is", "the" }; + + Token[] tokensA = { + newToken("A", 1), + newToken("is", 1), + newToken("The", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, false, + "A The B", + tokensA ); + + Token[] tokensB = { + newToken("A", 1), + newToken("is", 1), + newToken("The", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, true, + "A B", + tokensB ); + + Token[] tokensC = { + newToken("A", 1), + newToken("is", 1), + newToken("are", 0), + newToken("be", 0), + newToken("B", 1) + } ; + testPosInc(stopWords, false, + "A B", + tokensC ); + + Token[] tokensD = { + newToken("A", 1), + newToken("are", 1), + newToken("is", 0), + newToken("be", 0), + newToken("B", 1) + } ; + testPosInc(stopWords, false, + "A B", + tokensD ); + + Token[] tokensE = { + newToken("A", 1), + newToken("is", 1), + newToken("are", 0), + newToken("be", 0), + newToken("The", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, false, + "A The B", + tokensE ); + + Token[] tokensF = { + newToken("A", 1), + newToken("is", 1), + newToken("are", 0), + newToken("be", 0), + newToken("The", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, true, + "A B", + tokensF ); + } + }