### Eclipse Workspace Patch 1.0 #P WriteableTrunk-Lucene Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/package.html =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/package.html (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/package.html (working copy) @@ -3,8 +3,17 @@ +

Analyzer for Simplified Chinese, which indexes words. -

+ +WARNING: The status of the analyzers/smartcn analysis.cn package is experimental. The APIs +introduced here might change in the future and will not be supported anymore +in such a case. + +

Three analyzers are provided for Chinese, each of which treats Chinese text in a different way.

ChineseAnalyzer (in the analyzers/cn package): Index unigrams (individual Chinese characters) as a token. @@ -18,7 +27,7 @@
CJKAnalyzer: 我是－是中－中国－国人
SmartChineseAnalyzer: 我－是－中国－人

Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordSegmenter.java (working copy) @@ -26,6 +26,11 @@ /** * Segment a sentence of Chinese text into words. + *

+ * WARNING: The status of the analyzers/smartcn analysis.cn package is experimental. + * The APIs introduced here might change in the future and will not be + * supported anymore in such a case. + *

*/ class WordSegmenter { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordTokenFilter.java (working copy) @@ -30,6 +30,11 @@ /** * A {@link TokenFilter} that breaks sentences into words. + *

*/ public final class WordTokenFilter extends TokenFilter { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java (working copy) @@ -34,6 +34,11 @@ /** * SmartChineseAnalyzer Bigram dictionary. + *

*/ class BigramDictionary extends AbstractDictionary { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (working copy) @@ -31,6 +31,11 @@ *

* The output tokens can then be broken into words with {@link WordTokenFilter} *

+ *

*/ public final class SentenceTokenizer extends Tokenizer { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java (working copy) @@ -25,6 +25,11 @@ /** * Finds the optimal segmentation of a sentence into Chinese words + *

*/ public class HHMMSegmenter { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/package.html =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/package.html (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/package.html (working copy) @@ -1,5 +1,14 @@ +

SmartChineseAnalyzer Tokenizers and TokenFilters +

+ +WARNING: The status of the analyzers/smartcn analysis.cn package is experimental. The APIs +introduced here might change in the future and will not be supported anymore +in such a case. + +

Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java (working copy) @@ -36,6 +36,11 @@ /** * SmartChineseAnalyzer Word Dictionary * + *

*/ class WordDictionary extends AbstractDictionary { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordType.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordType.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/WordType.java (working copy) @@ -19,6 +19,11 @@ /** * Internal SmartChineseAnalyzer token type constants + *

*/ public class WordType { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java (working copy) @@ -22,6 +22,11 @@ *

* Used by {@link BiSegGraph} to maximize the segmentation with the Viterbi algorithm. *

+ *

*/ class PathNode implements Comparable { public double weight; Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/CharType.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/CharType.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/CharType.java (working copy) @@ -19,6 +19,11 @@ /** * Internal SmartChineseAnalyzer character type constants. + *

*/ public class CharType { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (working copy) @@ -21,6 +21,11 @@ /** * A pair of tokens in {@link SegGraph} + *

*/ class SegTokenPair { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (working copy) @@ -28,6 +28,11 @@ *

* For each start offset, a list of possible tokens is stored. *

+ *

*/ class SegGraph { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java (working copy) @@ -46,8 +46,12 @@ *

  * analysis.data.dir=D:/path/to/analysis-data/
  *

+ *

* - * */ public class AnalyzerProfile { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java (working copy) @@ -26,6 +26,11 @@ *

* Contains methods for dealing with GB2312 encoding. *

+ *

*/ abstract class AbstractDictionary { /** Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java (working copy) @@ -23,6 +23,11 @@ /** * SmartChineseAnalyzer internal token + *

*/ public class SegToken { /** Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/SmartChineseAnalyzer.java (working copy) @@ -68,6 +68,11 @@ *

  * analysis.data.dir=D:/path/to/analysis-data/
  *

+ *

*/ public class SmartChineseAnalyzer extends Analyzer { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenFilter.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenFilter.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenFilter.java (working copy) @@ -25,6 +25,11 @@ * Filters a {@link SegToken} by converting full-width latin to half-width, then lowercasing latin. * Additionally, all punctuation is converted into {@link Utility#COMMON_DELIMITER} *

+ *

*/ public class SegTokenFilter { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/Utility.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/Utility.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/Utility.java (working copy) @@ -21,6 +21,11 @@ /** * SmartChineseAnalyzer utility constants and methods + *

*/ public class Utility { Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/package.html =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/package.html (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/package.html (working copy) @@ -1,5 +1,14 @@ +

SmartChineseAnalyzer Hidden Markov Model package +

+ +WARNING: The status of the analyzers/smartcn analysis.cn package is experimental. The APIs +introduced here might change in the future and will not be supported anymore +in such a case. + +

Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java (revision 808032) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java (working copy) @@ -31,6 +31,11 @@ *

* For each start offset, a list of possible token pairs is stored. *

+ *

*/ class BiSegGraph {