+ This package provides dictionary-driven lemmatization ("accurate stemming") + filter and analyzer for the Polish Language, driven by the + Morfologik library developed + by Dawid Weiss and Marcin Miłkowski. +
++ The MorfologikFilter yields one or more terms for each token. Each + of those terms is given the same position in the index. +
+ + diff --git a/modules/analysis/morfologik/src/java/overview.html b/modules/analysis/morfologik/src/java/overview.html new file mode 100644 index 0000000..6b67d0e --- /dev/null +++ b/modules/analysis/morfologik/src/java/overview.html @@ -0,0 +1,34 @@ + + + + + + + ++ This package provides dictionary-driven lemmatization ("accurate stemming") + filter and analyzer for the Polish Language, driven by the + Morfologik library developed + by Dawid Weiss and Marcin Miłkowski. +
++ The MorfologikFilter yields one or more terms for each token. Each + of those terms is given the same position in the index. +
+ + diff --git a/modules/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java b/modules/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java new file mode 100644 index 0000000..9d9f146 --- /dev/null +++ b/modules/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikAnalyzer.java @@ -0,0 +1,71 @@ +// -*- c-basic-offset: 2 -*- +package org.apache.lucene.analysis.morfologik; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.StringReader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + +public class TestMorfologikAnalyzer extends BaseTokenStreamTestCase { + + private Analyzer getTestAnalyzer() { + return new MorfologikAnalyzer(TEST_VERSION_CURRENT); + } + + /** Test stemming of single tokens with Morfologik library. */ + public final void testSingleTokens() throws IOException { + Analyzer a = getTestAnalyzer(); + assertAnalyzesToReuse(a, "a", new String[] { "a" }); + assertAnalyzesToReuse(a, "liście", new String[] { "lista", "list", "liść" }); + assertAnalyzesToReuse(a, "danych", new String[] { "dać", "dane", "dany" }); + assertAnalyzesToReuse(a, "ęóąśłżźćń", new String[] { "ęóąśłżźćń" }); + } + + /** Test stemming of multiple tokens and proper term metrics. */ + public final void testMultipleTokens() throws IOException { + Analyzer a = getTestAnalyzer(); + assertAnalyzesToReuse( + a, + "liście danych", + new String[] { "lista", "list", "liść", "dać", "dane", "dany" }, + new int[] { 0, 0, 0, 7, 7, 7 }, + new int[] { 6, 6, 6, 13, 13, 13 }, + new int[] { 1, 0, 0, 1, 0, 0 }); + } + + /** Test reuse of MorfologikFilter with leftover stems. */ + public final void testLeftoverStems() throws IOException { + Analyzer a = getTestAnalyzer(); + TokenStream ts_1 = a.reusableTokenStream("dummy", new StringReader("liście")); + CharTermAttribute termAtt_1 = ts_1.getAttribute(CharTermAttribute.class); + ts_1.reset(); + ts_1.incrementToken(); + assertEquals("first stream", "lista", termAtt_1.toString()); + + TokenStream ts_2 = a.reusableTokenStream("dummy", new StringReader("danych")); + CharTermAttribute termAtt_2 = ts_2.getAttribute(CharTermAttribute.class); + ts_2.reset(); + ts_2.incrementToken(); + assertEquals("second stream", "dać", termAtt_2.toString()); + } +}