Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (revision 831682) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (working copy) @@ -39,11 +39,6 @@ */ public final class SentenceTokenizer extends Tokenizer { - /** - * End of sentence punctuation: 。，！？；,!?; - */ - private final static String PUNCTION = "。，！？；,!?;"; - private final StringBuilder buffer = new StringBuilder(); private int tokenStart = 0, tokenEnd = 0; @@ -86,12 +81,12 @@ while (true) { if (ci == -1) { break; - } else if (PUNCTION.indexOf(ch) != -1) { + } else if (isChinesePunctuation(ch)) { // End of a sentence buffer.append(ch); tokenEnd++; break; - } else if (atBegin && Utility.SPACES.indexOf(ch) != -1) { + } else if (atBegin && isChineseSpace(ch)) { tokenStart++; tokenEnd++; ci = input.read(); @@ -104,8 +99,7 @@ ci = input.read(); ch = (char) ci; // Two spaces, such as CR, LF - if (Utility.SPACES.indexOf(ch) != -1 - && Utility.SPACES.indexOf(pch) != -1) { + if (isChineseSpace(ch) && isChineseSpace(pch)) { // buffer.append(ch); tokenEnd++; break; @@ -115,7 +109,13 @@ if (buffer.length() == 0) return false; else { - termAtt.setTermBuffer(buffer.toString()); + final int length = buffer.length(); + char termBuffer[] = termAtt.termBuffer(); + if (termBuffer.length < length) + termBuffer = termAtt.resizeTermBuffer(length); + + buffer.getChars(0, length, termBuffer, 0); + termAtt.setTermLength(length); offsetAtt.setOffset(correctOffset(tokenStart), correctOffset(tokenEnd)); typeAtt.setType("sentence"); return true; @@ -131,4 +131,33 @@ super.reset(input); reset(); } + + private boolean isChinesePunctuation(int ch) { + switch(ch) { + case '。': + case '，': + case '！': + case '？': + case '；': + case ',': + case '!': + case '?': + return true; + default: + return false; + } + } + + private boolean isChineseSpace(int ch) { + switch(ch) { + case ' ': + case '　': + case '\t': + case '\r': + case '\n': + return true; + default: + return false; + } + } } Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java (revision 831717) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BiSegGraph.java (working copy) @@ -18,10 +18,7 @@ package org.apache.lucene.analysis.cn.smart.hhmm; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.lucene.analysis.cn.smart.Utility; @@ -38,14 +35,16 @@ */ class BiSegGraph { - private Map> tokenPairListTable = new HashMap>(); - + private final TokenGraph table; private List segTokenList; private static BigramDictionary bigramDict = BigramDictionary.getInstance(); public BiSegGraph(SegGraph segGraph) { + // get the list of tokens ordered and indexed segTokenList = segGraph.makeIndex(); + SegToken lastToken = segTokenList.get(segTokenList.size() - 1); + table = new TokenGraph(lastToken.index + 1); generateBiSegGraph(segGraph); } @@ -60,8 +59,7 @@ int next; char[] idBuffer; - // get the list of tokens ordered and indexed - segTokenList = segGraph.makeIndex(); + // Because the beginning position of startToken is -1, therefore startToken can be obtained when key = -1 int key = -1; List nextTokens = null; @@ -71,7 +69,8 @@ List tokenList = segGraph.getStartList(key); // Calculate all tokens for a given key. - for (SegToken t1 : tokenList) { + for (int i = 0; i < tokenList.size(); i++) { + SegToken t1 = tokenList.get(i); oneWordFreq = t1.weight; next = t1.endOffset; nextTokens = null; @@ -89,15 +88,11 @@ if (nextTokens == null) { break; } - for (SegToken t2 : nextTokens) { - idBuffer = new char[t1.charArray.length + t2.charArray.length + 1]; - System.arraycopy(t1.charArray, 0, idBuffer, 0, t1.charArray.length); - idBuffer[t1.charArray.length] = BigramDictionary.WORD_SEGMENT_CHAR; - System.arraycopy(t2.charArray, 0, idBuffer, - t1.charArray.length + 1, t2.charArray.length); + for (int j = 0; j < nextTokens.size(); j++) { + SegToken t2 = nextTokens.get(j); // Two linked Words frequency - wordPairFreq = bigramDict.getFrequency(idBuffer); + wordPairFreq = bigramDict.getFrequency(t1.charArray, t2.charArray); // Smoothing @@ -109,8 +104,7 @@ + (1.0 - smooth) * ((1.0 - tinyDouble) * wordPairFreq / (1.0 + oneWordFreq) + tinyDouble)); - SegTokenPair tokenPair = new SegTokenPair(idBuffer, t1.index, - t2.index, weight); + SegTokenPair tokenPair = new SegTokenPair(t1.index, t2.index, weight); this.addSegTokenPair(tokenPair); } } @@ -127,7 +121,7 @@ * @return true if a token pair exists */ public boolean isToExist(int to) { - return tokenPairListTable.get(Integer.valueOf(to)) != null; + return table.isExists(to); } /** @@ -137,7 +131,7 @@ * @return {@link List} of token pairs. */ public List getToList(int to) { - return tokenPairListTable.get(to); + return table.get(to); } /** @@ -146,23 +140,15 @@ * @param tokenPair {@link SegTokenPair} */ public void addSegTokenPair(SegTokenPair tokenPair) { - int to = tokenPair.to; - if (!isToExist(to)) { - ArrayList newlist = new ArrayList(); - newlist.add(tokenPair); - tokenPairListTable.put(to, newlist); - } else { - List tokenPairList = tokenPairListTable.get(to); - tokenPairList.add(tokenPair); - } + table.add(tokenPair.to, tokenPair); } /** - * Get the number of {@link SegTokenPair} entries in the table. - * @return number of {@link SegTokenPair} entries + * Get the number of mappings in the table. + * @return table size */ public int getToCount() { - return tokenPairListTable.size(); + return table.size(); } /** @@ -183,7 +169,8 @@ double minWeight = Double.MAX_VALUE; SegTokenPair minEdge = null; - for (SegTokenPair edge : edges) { + for (int i = 0; i < edges.size(); i++) { + SegTokenPair edge = edges.get(i); weight = edge.weight; PathNode preNode = path.get(edge.from); if (preNode.weight + weight < minWeight) { @@ -206,14 +193,13 @@ rpath.add(current); while (current != 0) { - PathNode currentPathNode = (PathNode) path.get(current); + PathNode currentPathNode = path.get(current); preNode = currentPathNode.preNode; - rpath.add(Integer.valueOf(preNode)); + rpath.add(preNode); current = preNode; } for (int j = rpath.size() - 1; j >= 0; j--) { - Integer idInteger = (Integer) rpath.get(j); - int id = idInteger.intValue(); + int id = rpath.get(j); SegToken t = segTokenList.get(id); resultPath.add(t); } @@ -223,13 +209,20 @@ public String toString() { StringBuilder sb = new StringBuilder(); - Collection> values = tokenPairListTable.values(); - for (ArrayList segList : values) { - for (SegTokenPair pair : segList) { - sb.append(pair + "\n"); - } - } + for (int i = 0; i < table.getMax(); i++) + if (table.isExists(i)) + for (SegTokenPair pair : table.get(i)) + sb.append(pair + "\n"); + return sb.toString(); } + + void reset(SegGraph segGraph) { + // get the list of tokens ordered and indexed + segTokenList = segGraph.makeIndex(); + SegToken lastToken = segTokenList.get(segTokenList.size() - 1); + table.reset(lastToken.index + 1); + generateBiSegGraph(segGraph); + } } Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java (revision 831682) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java (working copy) @@ -60,8 +60,6 @@ private int max = 0; - private int repeat = 0; - // static Logger log = Logger.getLogger(BigramDictionary.class); public synchronized static BigramDictionary getInstance() { @@ -250,12 +248,10 @@ hash2 = PRIME_BIGRAM_LENGTH + hash2; int index = hash1; int i = 1; - repeat++; while (bigramHashTable[index] != 0 && bigramHashTable[index] != hashId && i < PRIME_BIGRAM_LENGTH) { index = (hash1 + i * hash2) % PRIME_BIGRAM_LENGTH; i++; - repeat++; if (i > max) max = i; } @@ -273,5 +269,94 @@ return frequencyTable[index]; return 0; } + + /** + * 32-bit FNV Hash Function (on bigrams) + * + * @param left left side of bigram + * @param right right side of bigram + * @return hashcode + */ + private long hash1(char left[], char right[]) { + final long p = 1099511628211L; + long hash = 0xcbf29ce484222325L; + for (int i = 0; i < left.length; i++) { + char d = left[i]; + hash = (hash ^ (d & 0x00FF)) * p; + hash = (hash ^ (d >> 8)) * p; + } + + hash = (hash ^ (BigramDictionary.WORD_SEGMENT_CHAR & 0x00FF)) * p; + hash = (hash ^ (BigramDictionary.WORD_SEGMENT_CHAR >> 8)) * p; + + for (int i = 0; i < right.length; i++) { + char d = right[i]; + hash = (hash ^ (d & 0x00FF)) * p; + hash = (hash ^ (d >> 8)) * p; + } + return hash; + } + + /** + * djb2 hash algorithm (k=33) (on bigrams) + * + * @param left left side of bigram + * @param right right side of bigram + * @return hashcode + */ + private int hash2(char left[], char right[]) { + int hash = 5381; + /* hash 33 + c */ + for (int i = 0; i < left.length; i++) { + char d = left[i]; + hash = ((hash << 5) + hash) + d & 0x00FF; + hash = ((hash << 5) + hash) + d >> 8; + } + + hash = ((hash << 5) + hash) + BigramDictionary.WORD_SEGMENT_CHAR & 0x00FF; + hash = ((hash << 5) + hash) + BigramDictionary.WORD_SEGMENT_CHAR >> 8; + + for (int i = 0; i < right.length; i++) { + char d = right[i]; + hash = ((hash << 5) + hash) + d & 0x00FF; + hash = ((hash << 5) + hash) + d >> 8; + } + return hash; + } + + /* + * lookup the index into the frequency array for a bigram + */ + private int getBigramItemIndex(char left[], char right[]) { + long hashId = hash1(left, right); + int hash1 = (int) (hashId % PRIME_BIGRAM_LENGTH); + int hash2 = hash2(left, right) % PRIME_BIGRAM_LENGTH; + if (hash1 < 0) + hash1 = PRIME_BIGRAM_LENGTH + hash1; + if (hash2 < 0) + hash2 = PRIME_BIGRAM_LENGTH + hash2; + int index = hash1; + int i = 1; + while (bigramHashTable[index] != 0 && bigramHashTable[index] != hashId + && i < PRIME_BIGRAM_LENGTH) { + index = (hash1 + i * hash2) % PRIME_BIGRAM_LENGTH; + i++; + if (i > max) + max = i; + } + // System.out.println(i - 1); + + if (i < PRIME_BIGRAM_LENGTH && bigramHashTable[index] == hashId) { + return index; + } else + return -1; + } + + public int getFrequency(char left[], char right[]) { + int index = getBigramItemIndex(left, right); + if (index != -1) + return frequencyTable[index]; + return 0; + } } Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java (revision 831682) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/HHMMSegmenter.java (working copy) @@ -35,6 +35,11 @@ public class HHMMSegmenter { private static WordDictionary wordDict = WordDictionary.getInstance(); + + // reusable word graph + private SegGraph segGraph = new SegGraph(10); + // reusable bigram graph + private BiSegGraph biSegGraph; /** * Create the {@link SegGraph} for a sentence. @@ -54,7 +59,7 @@ int wordType; char[] charArray; - SegGraph segGraph = new SegGraph(); + segGraph.reset(length); while (i < length) { hasFullWidth = false; switch (charTypeArray[i]) { @@ -202,7 +207,10 @@ */ public List process(String sentence) { SegGraph segGraph = createSegGraph(sentence); - BiSegGraph biSegGraph = new BiSegGraph(segGraph); + if (biSegGraph == null) + biSegGraph = new BiSegGraph(segGraph); + else + biSegGraph.reset(segGraph); List shortPath = biSegGraph.getShortPath(); return shortPath; } Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (revision 831682) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (working copy) @@ -18,9 +18,7 @@ package org.apache.lucene.analysis.cn.smart.hhmm; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; /** * Graph representing possible tokens at each start offset in the sentence. @@ -37,19 +35,26 @@ /** * Map of start offsets to ArrayList of tokens at that position + * Note: offset i is stored as array index i + 1. + * This is because the sentence start token is stored at position -1 */ - private Map> tokenListTable = new HashMap>(); + private final TokenGraph table; - private int maxStart = -1; - /** + * Create a new SegGraph capable of representing a sentence of maxSize. + */ + SegGraph(int maxSize) { + // sentence size, plus sentence start marker, plus sentence end marker + table = new TokenGraph(maxSize + 2); + } + /** * Returns true if a mapping for the specified start offset exists * * @param s startOffset * @return true if there are tokens for the startOffset */ public boolean isStartExist(int s) { - return tokenListTable.get(s) != null; + return table.isExists(s + 1); } /** @@ -59,7 +64,7 @@ * @return List of tokens at the specified start offset. */ public List getStartList(int s) { - return tokenListTable.get(s); + return table.get(s + 1); } /** @@ -68,7 +73,7 @@ * @return maximum start offset, or -1 if the map is empty. */ public int getMaxStart() { - return maxStart; + return table.getMax() - 1; } /** @@ -77,13 +82,14 @@ */ public List makeIndex() { List result = new ArrayList(); - int s = -1, count = 0, size = tokenListTable.size(); + int s = -1, count = 0, size = table.size(); List tokenList; short index = 0; while (count < size) { if (isStartExist(s)) { - tokenList = tokenListTable.get(s); - for (SegToken st : tokenList) { + tokenList = getStartList(s); + for (int i = 0; i < tokenList.size(); i++) { + SegToken st = tokenList.get(i); st.index = index; result.add(st); index++; @@ -100,17 +106,7 @@ * @param token {@link SegToken} */ public void addToken(SegToken token) { - int s = token.startOffset; - if (!isStartExist(s)) { - ArrayList newlist = new ArrayList(); - newlist.add(token); - tokenListTable.put(s, newlist); - } else { - List tokenList = tokenListTable.get(s); - tokenList.add(token); - } - if (s > maxStart) - maxStart = s; + table.add(token.startOffset + 1, token); } /** @@ -120,14 +116,14 @@ */ public List toTokenList() { List result = new ArrayList(); - int s = -1, count = 0, size = tokenListTable.size(); + int s = -1, count = 0, size = table.size(); List tokenList; while (count < size) { if (isStartExist(s)) { - tokenList = tokenListTable.get(s); - for (SegToken st : tokenList) { - result.add(st); + tokenList = getStartList(s); + for (int i = 0; i < tokenList.size(); i++) { + result.add(tokenList.get(i)); } count++; } @@ -144,4 +140,8 @@ } return sb.toString(); } + + void reset(int size) { + table.reset(size + 2); + } } Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java =================================================================== --- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (revision 831717) +++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (working copy) @@ -17,8 +17,6 @@ package org.apache.lucene.analysis.cn.smart.hhmm; -import java.util.Arrays; - /** * A pair of tokens in {@link SegGraph} *

@@ -28,9 +26,6 @@ *

*/ class SegTokenPair { - - public char[] charArray; - /** * index of the first token in {@link SegGraph} */ @@ -43,8 +38,7 @@ public double weight; - public SegTokenPair(char[] idArray, int from, int to, double weight) { - this.charArray = idArray; + public SegTokenPair(int from, int to, double weight) { this.from = from; this.to = to; this.weight = weight; @@ -56,9 +50,6 @@ public int hashCode() { final int prime = 31; int result = 1; - for(int i=0;i { + private List> table = new ArrayList>(); + private int size = 0; + private int max = 0; + + /** + * Create a new TokenGraph capable of holding size mappings. + */ + TokenGraph(int size) { + reset(size); + } + + /** + * Add an item at the specified position + */ + void add(int position, E item) { + List list = table.get(position); + if (list.isEmpty()) + size++; + list.add(item); + max = Math.max(max, position); + } + + /** + * Get the list of items at the specified position + * @return {@link List} of items at this position. + * The list will be empty if no mappings exist. + */ + List get(int position) { + return table.get(position); + } + + boolean isExists(int position) { + return !table.get(position).isEmpty(); + } + + /** + * Get the number of mappings in this graph. + */ + int size() { + return size; + } + + /** + * Get the maximum position that has a mapping in this graph. + */ + int getMax() { + return max; + } + + /** + * Clear the graph, resetting it to a new empty graph of size + */ + void reset(int size) { + if (table.size() >= size) { + for (int i = 0; i < size; i++) + table.get(i).clear(); + } else { + for (int i = 0; i < table.size(); i++) + table.get(i).clear(); + for (int i = table.size(); i < size; i++) + table.add(new ArrayList()); + } + this.size = 0; + this.max = 0; + } +} Property changes on: contrib\analyzers\smartcn\src\java\org\apache\lucene\analysis\cn\smart\hhmm\TokenGraph.java ___________________________________________________________________ Added: svn:eol-style + native Index: contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java =================================================================== --- contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (revision 831682) +++ contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (working copy) @@ -94,6 +94,14 @@ assertAnalyzesTo(ca, sentence, result, startOffsets, endOffsets, posIncr); } + /* + * Test analyzing an empty document + */ + public void testEmpty() throws Exception { + Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT); + assertAnalyzesTo(ca, "", new String[] {}); + } + public void testChineseAnalyzer() throws Exception { Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true); String sentence = "我购买了道具和服装。";