();
BufferedReader br = null;
try {
- if (reader instanceof BufferedReader) {
- br = (BufferedReader) reader;
- } else {
- br = new BufferedReader(reader);
- }
+ br = getBufferedReader(reader);
String line = null;
while ((line = br.readLine()) != null) {
int comment = line.indexOf('|');
@@ -247,10 +145,30 @@ public class WordlistLoader {
if (words[i].length() > 0) result.add(words[i]);
}
} finally {
- if (br != null) br.close();
+ IOUtils.close(br);
}
return result;
}
+
+ /**
+ * Reads stopwords from a stopword list in Snowball format.
+ *
+ * The snowball format is the following:
+ *
+ * - Lines may contain multiple words separated by whitespace.
+ *
- The comment character is the vertical line (|).
+ *
- Lines may contain trailing comments.
+ *
+ *
+ *
+ * @param reader Reader containing a Snowball stopword list
+ * @param result the {@link CharArraySet} to fill with the readers words
+ * @param matchVersion the Lucene {@link Version}
+ * @return A {@link CharArraySet} with the reader's words
+ */
+ public static CharArraySet getSnowballWordSet(Reader reader, Version matchVersion) throws IOException {
+ return getSnowballWordSet(reader, new CharArraySet(matchVersion, INITITAL_CAPACITY, false));
+ }
/**
@@ -261,24 +179,24 @@ public class WordlistLoader {
* @return stem dictionary that overrules the stemming algorithm
* @throws IOException
*/
- public static HashMap getStemDict(File wordstemfile) throws IOException {
- if (wordstemfile == null)
- throw new NullPointerException("wordstemfile may not be null");
- final HashMap result = new HashMap();
+ public static CharArrayMap getStemDict(Reader reader, CharArrayMap result) throws IOException {
BufferedReader br = null;
-
try {
- br = new BufferedReader(new FileReader(wordstemfile));
+ br = getBufferedReader(reader);
String line;
while ((line = br.readLine()) != null) {
String[] wordstem = line.split("\t", 2);
result.put(wordstem[0], wordstem[1]);
}
} finally {
- if(br != null)
- br.close();
+ IOUtils.close(br);
}
return result;
}
-
+
+ private static BufferedReader getBufferedReader(Reader reader) {
+ return (reader instanceof BufferedReader) ? (BufferedReader) reader
+ : new BufferedReader(reader);
+ }
+
}
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArraySet.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArraySet.java
index 8983ead..9cb0757 100755
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArraySet.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharArraySet.java
@@ -46,7 +46,7 @@ public class TestCharArraySet extends LuceneTestCase {
public void testNonZeroOffset() {
String[] words={"Hello","World","this","is","a","test"};
char[] findme="xthisy".toCharArray();
- CharArraySet set=new CharArraySet(TEST_VERSION_CURRENT, 10,true);
+ CharArraySet set= new CharArraySet(TEST_VERSION_CURRENT, 10, true);
set.addAll(Arrays.asList(words));
assertTrue(set.contains(findme, 1, 4));
assertTrue(set.contains(new String(findme,1,4)));
diff --git a/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestWordlistLoader.java b/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestWordlistLoader.java
index 74356c4..a9634f6 100644
--- a/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestWordlistLoader.java
+++ b/modules/analysis/common/src/test/org/apache/lucene/analysis/util/TestWordlistLoader.java
@@ -20,8 +20,6 @@ package org.apache.lucene.analysis.util;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.lucene.util.LuceneTestCase;
@@ -31,22 +29,22 @@ public class TestWordlistLoader extends LuceneTestCase {
public void testWordlistLoading() throws IOException {
String s = "ONE\n two \nthree";
- HashSet wordSet1 = WordlistLoader.getWordSet(new StringReader(s));
+ CharArraySet wordSet1 = WordlistLoader.getWordSet(new StringReader(s), TEST_VERSION_CURRENT);
checkSet(wordSet1);
- HashSet wordSet2 = WordlistLoader.getWordSet(new BufferedReader(new StringReader(s)));
+ CharArraySet wordSet2 = WordlistLoader.getWordSet(new BufferedReader(new StringReader(s)), TEST_VERSION_CURRENT);
checkSet(wordSet2);
}
public void testComments() throws Exception {
String s = "ONE\n two \nthree\n#comment";
- HashSet wordSet1 = WordlistLoader.getWordSet(new StringReader(s), "#");
+ CharArraySet wordSet1 = WordlistLoader.getWordSet(new StringReader(s), "#", TEST_VERSION_CURRENT);
checkSet(wordSet1);
assertFalse(wordSet1.contains("#comment"));
assertFalse(wordSet1.contains("comment"));
}
- private void checkSet(HashSet wordset) {
+ private void checkSet(CharArraySet wordset) {
assertEquals(3, wordset.size());
assertTrue(wordset.contains("ONE")); // case is not modified
assertTrue(wordset.contains("two")); // surrounding whitespace is removed
@@ -68,7 +66,7 @@ public class TestWordlistLoader extends LuceneTestCase {
" two \n" + // stopword with leading/trailing space
" three four five \n" + // multiple stopwords
"six seven | comment\n"; //multiple stopwords + comment
- Set wordset = WordlistLoader.getSnowballWordSet(new StringReader(s));
+ CharArraySet wordset = WordlistLoader.getSnowballWordSet(new StringReader(s), TEST_VERSION_CURRENT);
assertEquals(7, wordset.size());
assertTrue(wordset.contains("ONE"));
assertTrue(wordset.contains("two"));
diff --git a/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java b/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
index f078b6a..8de8b34 100644
--- a/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
+++ b/modules/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.en.PorterStemFilter;
+import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -66,7 +67,7 @@ public final class SmartChineseAnalyzer extends Analyzer {
* Returns an unmodifiable instance of the default stop-words set.
* @return an unmodifiable instance of the default stop-words set.
*/
- public static Set getDefaultStopSet(){
+ public static CharArraySet getDefaultStopSet(){
return DefaultSetHolder.DEFAULT_STOP_SET;
}
@@ -75,7 +76,7 @@ public final class SmartChineseAnalyzer extends Analyzer {
* accesses the static final set the first time.;
*/
private static class DefaultSetHolder {
- static final Set DEFAULT_STOP_SET;
+ static final CharArraySet DEFAULT_STOP_SET;
static {
try {
@@ -87,13 +88,14 @@ public final class SmartChineseAnalyzer extends Analyzer {
}
}
- static Set loadDefaultStopWordSet() throws IOException {
+ static CharArraySet loadDefaultStopWordSet() throws IOException {
InputStream stream = SmartChineseAnalyzer.class
.getResourceAsStream(DEFAULT_STOPWORD_FILE);
try {
InputStreamReader reader = new InputStreamReader(stream, "UTF-8");
// make sure it is unmodifiable as we expose it in the outer class
- return Collections.unmodifiableSet(WordlistLoader.getWordSet(reader, STOPWORD_FILE_COMMENT));
+ return CharArraySet.unmodifiableSet(WordlistLoader.getWordSet(reader,
+ STOPWORD_FILE_COMMENT, Version.LUCENE_CURRENT));
} finally {
stream.close();
}
diff --git a/modules/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java b/modules/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
index 8dc589a..4266e11 100644
--- a/modules/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
+++ b/modules/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
@@ -34,6 +34,7 @@ import org.apache.lucene.analysis.stempel.StempelFilter;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
import org.egothor.stemmer.Trie;
@@ -68,8 +69,8 @@ public final class PolishAnalyzer extends StopwordAnalyzerBase {
static {
try {
- DEFAULT_STOP_SET = WordlistLoader.getWordSet(PolishAnalyzer.class,
- DEFAULT_STOPWORD_FILE);
+ DEFAULT_STOP_SET = WordlistLoader.getWordSet(IOUtils.getReader(PolishAnalyzer.class,
+ DEFAULT_STOPWORD_FILE, IOUtils.CHARSET_UTF8), "#", Version.LUCENE_CURRENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)