Index: src/java/org/apache/lucene/analysis/StopFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/StopFilter.java (revision 544097) +++ src/java/org/apache/lucene/analysis/StopFilter.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.HashSet; +import java.util.LinkedList; import java.util.Set; /** @@ -29,6 +30,7 @@ private final Set stopWords; private final boolean ignoreCase; + private final boolean removeStopwordCollocates; /** * Construct a token stream filtering the given input. @@ -43,25 +45,50 @@ * TokenStream that are named in the array of words. */ public StopFilter(TokenStream in, String[] stopWords, boolean ignoreCase) { + this(in, stopWords, ignoreCase, false ); + } + + + /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + */ + public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase) + { + this(input, stopWords, ignoreCase, false ); + } + + /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + * @param removeStopwordCollocates -When stopword is included in token at the same position, everything is removed. + */ + public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase, boolean removeStopwordCollocates) + { + super(input); + this.ignoreCase = ignoreCase; + this.stopWords = stopWords; + this.removeStopwordCollocates = removeStopwordCollocates ; + } + + /** + * Construct a token stream filtering the given input. + * @param input + * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased + * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words + * @param removeStopwordCollocates -When stopword is included in token at the same position, everything is removed. + */ + public StopFilter(TokenStream in, String[] stopWords, boolean ignoreCase, boolean removeStopwordCollocates) { super(in); this.ignoreCase = ignoreCase; this.stopWords = makeStopSet(stopWords, ignoreCase); + this.removeStopwordCollocates = removeStopwordCollocates ; } - - /** - * Construct a token stream filtering the given input. - * @param input - * @param stopWords The set of Stop Words, as Strings. If ignoreCase is true, all strings should be lower cased - * @param ignoreCase -Ignore case when stopping. The stopWords set must be setup to contain only lower case words - */ - public StopFilter(TokenStream input, Set stopWords, boolean ignoreCase) - { - super(input); - this.ignoreCase = ignoreCase; - this.stopWords = stopWords; - } - /** * Constructs a filter which removes words from the input * TokenStream that are named in the Set. @@ -88,7 +115,7 @@ /** * - * @param stopWords + * @param stopWords * @param ignoreCase If true, all words are lower cased first. * @return a Set containing the words */ @@ -99,18 +126,110 @@ return stopTable; } + private final LinkedList inQueue = new LinkedList(); + private final LinkedList outQueue = new LinkedList(); + /** + * Read a token from the buffered input stream. + * @return null at EOS + */ + protected Token read() throws IOException { + if (inQueue.isEmpty()) { + Token t = input.next(); + return t; + } + return inQueue.removeFirst(); + } + /** + * Push a token back into the buffered input stream, such that it will + * be returned by a future call to read() + */ + protected void pushBack(Token t) { + inQueue.addFirst(t); + } + + /** + * Write a token to the buffered output stream + */ + protected void write(Token t) { + outQueue.addLast(t); + } + + /** + * removeStopwordCollocates + * + * @return Returns the next input Token + * @throws IOException + */ + private final Token removeStopwordCollocatesNext() throws IOException { + while (true) { + if (!outQueue.isEmpty()) { + return outQueue.removeFirst(); + } + Token token = read(); + if (null == token) { + return null; + } + + Token next = read(); + + boolean stopword = false; + + LinkedList checkQueue = new LinkedList(); + checkQueue.addLast(token); + + String termText = ignoreCase ? token.termText.toLowerCase() + : token.termText; + if (stopWords.contains(termText)) { + stopword = true; + } + + while (next != null && next.getPositionIncrement() == 0) { + checkQueue.addLast(next); + termText = ignoreCase ? next.termText.toLowerCase() : next.termText; + if (stopWords.contains(termText)) { + stopword = true; + } + next = read(); + } + if (!stopword) { + for (Token out : checkQueue) { + write(out); + } + } + if (next != null) { + pushBack(next); + } + } + } + /** * Returns the next input Token whose termText() is not a stop word. */ public final Token next() throws IOException { - // return the first non-stop word found - for (Token token = input.next(); token != null; token = input.next()) - { - String termText = ignoreCase ? token.termText.toLowerCase() : token.termText; - if (!stopWords.contains(termText)) + if (removeStopwordCollocates) { + return removeStopwordCollocatesNext(); + } else { + boolean prev_stopword = false; + int org_posInc = 0; + // return the first non-stop word found + for (Token token = input.next(); token != null; token = input.next()) { + String termText = ignoreCase ? token.termText.toLowerCase() + : token.termText; + if (stopWords.contains(termText)) { + if (token.getPositionIncrement() != 0) { + prev_stopword = true; + org_posInc = token.getPositionIncrement(); + } + } else { + if (prev_stopword && token.getPositionIncrement() == 0) { + prev_stopword = false; + token.setPositionIncrement(org_posInc); + } return token; + } + } + // reached EOS -- return null + return null; } - // reached EOS -- return null - return null; } } Index: src/test/org/apache/lucene/analysis/TestStopFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/TestStopFilter.java (revision 544097) +++ src/test/org/apache/lucene/analysis/TestStopFilter.java (working copy) @@ -18,6 +18,8 @@ import java.io.IOException; import java.io.StringReader; +import java.util.Arrays; +import java.util.Iterator; import junit.framework.TestCase; @@ -44,5 +46,177 @@ assertEquals("Now", stream.next().termText()); assertEquals(null,stream.next()); } + + + public Token newToken(String termText, int pos) { + Token token = new Token(termText, 0, 0); + token.setPositionIncrement(pos); + return token; + } + public void testPosInc(String[] stopWords, boolean ignoreCase, + final String expectation, final Token[] tokens) throws IOException { + + final Iterator token = Arrays.asList(tokens).iterator(); + + final TokenStream ts = new StopFilter(new TokenStream() { + public Token next() { + return token.hasNext() ? token.next() : null; + } + }, stopWords, ignoreCase, true); + + final String result = tokenStream2String(ts); + assertEquals(expectation + " != " + result, expectation, result); + // System.out.println(result) ; + } + + public void testPosInc(String[] stopWords, boolean ignoreCase, + boolean removeStopwordCollocates, final Token[] in_tokens, + final Token[] out_tokens) throws IOException { + + final Iterator token = Arrays.asList(in_tokens).iterator(); + + final TokenStream ts = new StopFilter(new TokenStream() { + public Token next() { + return token.hasNext() ? token.next() : null; + } + }, stopWords, ignoreCase, removeStopwordCollocates); + + Token stop_out = null; + int i = 0; + + for (stop_out = ts.next(), i = 0; null != stop_out; stop_out = ts.next(), i++) { + if (!stop_out.termText().equals(out_tokens[i].termText())) { + fail(stop_out.termText() + "!=" + out_tokens[i].termText()); + break; + } else if (stop_out.getPositionIncrement() != out_tokens[i] + .getPositionIncrement()) { + fail("fail position increment"); + break; + } + } + } + + public static String tokenStream2String(TokenStream input) throws IOException { + StringBuffer output = new StringBuffer(); + Token token = input.next(); + if (null != token) { + output.append(token.termText()); + } + for (token = input.next(); null != token; token = input.next()) { + output.append(" ").append(token.termText()); + } + input.close(); + return output.toString(); + } + + public void testPosIncs() throws IOException { + + String[] stopWords = new String[] { "is", "am", "the" }; + + Token[] in_tokensA = { + newToken("A", 1), + newToken("is", 1), + newToken("are", 0), + newToken("be", 0), + newToken("B", 1) + }; + Token[] out_tokensA = { + newToken("A", 1), + newToken("are", 1), + newToken("be", 0), + newToken("B", 1) + }; + testPosInc(stopWords, + false, + false, + in_tokensA, + out_tokensA); + + Token[] in_tokensB = { + newToken("A", 1), + newToken("are", 1), + newToken("is", 0), + newToken("be", 0), + newToken("am", 0), + newToken("B", 1) + }; + Token[] out_tokensB = { + newToken("A", 1), + newToken("are", 1), + newToken("be", 0), + newToken("B", 1) + }; + testPosInc(stopWords, + false, + false, + in_tokensB, + out_tokensB); + + Token[] in_tokensC = { + newToken("A", 1), + newToken("are", 1), + newToken("be", 0), + newToken("is", 0), + newToken("B", 1) + }; + Token[] out_tokensC = { + newToken("A", 1), + newToken("are", 1), + newToken("be", 0), + newToken("B", 1) + }; + testPosInc(stopWords, + false, + false, + in_tokensC, + out_tokensC); + + + Token[] in_tokensD = { + newToken("A", 1), + newToken("is", 1), + newToken("are", 0), + newToken("be", 0), + newToken("B", 1) + } ; + Token[] out_tokensD = { + newToken("A", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, + false, + true, + in_tokensD, + out_tokensD); + + Token[] in_tokensE = { + newToken("A", 1), + newToken("are", 1), + newToken("is", 0), + newToken("be", 0), + newToken("B", 1) + } ; + Token[] out_tokensE = { + newToken("A", 1), + newToken("B", 1) + } ; + testPosInc(stopWords, + false, + true, + in_tokensE, + out_tokensE); + + Token[] tokensF = { + newToken("A", 1), + newToken("are", 1), + newToken("is", 0), + newToken("be", 0), + newToken("B", 1) + } ; + testPosInc(stopWords, false, + "A B", + tokensF ); + } + }