shortPath = biSegGraph.getShortPath();
return shortPath;
}
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java (revision 831792)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/PathNode.java (working copy)
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.cn.smart.hhmm;
-
-/**
- * SmartChineseAnalyzer internal node representation
- *
- * Used by {@link BiSegGraph} to maximize the segmentation with the Viterbi algorithm.
- *
- *
- * WARNING: The status of the analyzers/smartcn analysis.cn.smart package is experimental.
- * The APIs and file formats introduced here might change in the future and will not be
- * supported anymore in such a case.
- *
- */
-class PathNode implements Comparable {
- public double weight;
-
- public int preNode;
-
- public int compareTo(PathNode pn) {
- if (weight < pn.weight)
- return -1;
- else if (weight == pn.weight)
- return 0;
- else
- return 1;
- }
-
- /**
- * @see java.lang.Object#hashCode()
- */
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + preNode;
- long temp;
- temp = Double.doubleToLongBits(weight);
- result = prime * result + (int) (temp ^ (temp >>> 32));
- return result;
- }
-
- /**
- * @see java.lang.Object#equals(java.lang.Object)
- */
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- PathNode other = (PathNode) obj;
- if (preNode != other.preNode)
- return false;
- if (Double.doubleToLongBits(weight) != Double
- .doubleToLongBits(other.weight))
- return false;
- return true;
- }
-}
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (revision 831792)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegGraph.java (working copy)
@@ -18,9 +18,7 @@
package org.apache.lucene.analysis.cn.smart.hhmm;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
/**
* Graph representing possible tokens at each start offset in the sentence.
@@ -37,19 +35,17 @@
/**
* Map of start offsets to ArrayList of tokens at that position
+ * Note: offset i is stored as array index i + 1.
+ * This is because the sentence start token is stored at position -1
*/
- private Map> tokenListTable = new HashMap>();
+ private final TokenGraph table;
- private int maxStart = -1;
-
/**
- * Returns true if a mapping for the specified start offset exists
- *
- * @param s startOffset
- * @return true if there are tokens for the startOffset
+ * Create a new SegGraph capable of representing a sentence of maxSize.
*/
- public boolean isStartExist(int s) {
- return tokenListTable.get(s) != null;
+ SegGraph(int maxSize) {
+ // sentence size, plus sentence start marker, plus sentence end marker
+ table = new TokenGraph(maxSize + 2);
}
/**
@@ -59,7 +55,7 @@
* @return List of tokens at the specified start offset.
*/
public List getStartList(int s) {
- return tokenListTable.get(s);
+ return table.get(s + 1);
}
/**
@@ -68,7 +64,7 @@
* @return maximum start offset, or -1 if the map is empty.
*/
public int getMaxStart() {
- return maxStart;
+ return table.getMax() - 1;
}
/**
@@ -77,13 +73,14 @@
*/
public List makeIndex() {
List result = new ArrayList();
- int s = -1, count = 0, size = tokenListTable.size();
+ int s = -1, count = 0, size = table.size();
List tokenList;
short index = 0;
while (count < size) {
- if (isStartExist(s)) {
- tokenList = tokenListTable.get(s);
- for (SegToken st : tokenList) {
+ tokenList = getStartList(s);
+ if (!tokenList.isEmpty()) {
+ for (int i = 0; i < tokenList.size(); i++) {
+ SegToken st = tokenList.get(i);
st.index = index;
result.add(st);
index++;
@@ -92,6 +89,7 @@
}
s++;
}
+
return result;
}
@@ -100,48 +98,19 @@
* @param token {@link SegToken}
*/
public void addToken(SegToken token) {
- int s = token.startOffset;
- if (!isStartExist(s)) {
- ArrayList newlist = new ArrayList();
- newlist.add(token);
- tokenListTable.put(s, newlist);
- } else {
- List tokenList = tokenListTable.get(s);
- tokenList.add(token);
- }
- if (s > maxStart)
- maxStart = s;
+ table.add(token.startOffset + 1, token);
}
- /**
- * Return a {@link List} of all tokens in the map, ordered by startOffset.
- *
- * @return {@link List} of all tokens in the map.
- */
- public List toTokenList() {
- List result = new ArrayList();
- int s = -1, count = 0, size = tokenListTable.size();
- List tokenList;
-
- while (count < size) {
- if (isStartExist(s)) {
- tokenList = tokenListTable.get(s);
- for (SegToken st : tokenList) {
- result.add(st);
- }
- count++;
- }
- s++;
- }
- return result;
- }
-
public String toString() {
- List tokenList = this.toTokenList();
+ List tokenList = makeIndex();
StringBuilder sb = new StringBuilder();
for (SegToken t : tokenList) {
sb.append(t + "\n");
}
return sb.toString();
}
+
+ void reset(int size) {
+ table.reset(size + 2);
+ }
}
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java (revision 831792)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegToken.java (working copy)
@@ -36,12 +36,12 @@
public char[] charArray;
/**
- * start offset into {@link #charArray}
+ * start offset into original sentence
*/
public int startOffset;
/**
- * end offset into {@link #charArray}
+ * end offset into original sentence
*/
public int endOffset;
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (revision 831792)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/SegTokenPair.java (working copy)
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.analysis.cn.smart.hhmm;
-
-import java.util.Arrays;
-
-/**
- * A pair of tokens in {@link SegGraph}
- *
- * WARNING: The status of the analyzers/smartcn analysis.cn.smart package is experimental.
- * The APIs and file formats introduced here might change in the future and will not be
- * supported anymore in such a case.
- *
- */
-class SegTokenPair {
-
- public char[] charArray;
-
- /**
- * index of the first token in {@link SegGraph}
- */
- public int from;
-
- /**
- * index of the second token in {@link SegGraph}
- */
- public int to;
-
- public double weight;
-
- public SegTokenPair(char[] idArray, int from, int to, double weight) {
- this.charArray = idArray;
- this.from = from;
- this.to = to;
- this.weight = weight;
- }
-
- /**
- * @see java.lang.Object#hashCode()
- */
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- for(int i=0;i>> 32));
- return result;
- }
-
- /**
- * @see java.lang.Object#equals(java.lang.Object)
- */
- public boolean equals(Object obj) {
- if (this == obj)
- return true;
- if (obj == null)
- return false;
- if (getClass() != obj.getClass())
- return false;
- SegTokenPair other = (SegTokenPair) obj;
- if (!Arrays.equals(charArray, other.charArray))
- return false;
- if (from != other.from)
- return false;
- if (to != other.to)
- return false;
- if (Double.doubleToLongBits(weight) != Double
- .doubleToLongBits(other.weight))
- return false;
- return true;
- }
-
-}
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/TokenGraph.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/TokenGraph.java (revision 0)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/TokenGraph.java (revision 0)
@@ -0,0 +1,95 @@
+package org.apache.lucene.analysis.cn.smart.hhmm;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A generic graph of items indexed by position.
+ */
+final class TokenGraph {
+ private List> table = new ArrayList>();
+ private int size = 0;
+ private int max = 0;
+
+ /**
+ * Create a new TokenGraph capable of holding size mappings.
+ */
+ TokenGraph(int size) {
+ reset(size);
+ }
+
+ /**
+ * Add an item at the specified position
+ */
+ void add(int position, E item) {
+ List list = table.get(position);
+ if (list.isEmpty())
+ size++;
+ list.add(item);
+ max = Math.max(max, position);
+ }
+
+ /**
+ * Get the list of items at the specified position
+ * @return {@link List} of items at this position.
+ * The list will be empty if no mappings exist.
+ */
+ List get(int position) {
+ return table.get(position);
+ }
+
+ /**
+ * True if items exist at the specified position
+ */
+ boolean isExists(int position) {
+ return !table.get(position).isEmpty();
+ }
+
+ /**
+ * Get the number of mappings in this graph.
+ */
+ int size() {
+ return size;
+ }
+
+ /**
+ * Get the maximum position that has a mapping in this graph.
+ */
+ int getMax() {
+ return max;
+ }
+
+ /**
+ * Clear the graph, resetting it to a new empty graph of size
+ */
+ void reset(int size) {
+ if (table.size() >= size) {
+ for (int i = 0; i < size; i++)
+ table.get(i).clear();
+ } else {
+ for (int i = 0; i < table.size(); i++)
+ table.get(i).clear();
+ for (int i = table.size(); i < size; i++)
+ table.add(new ArrayList());
+ }
+ this.size = 0;
+ this.max = 0;
+ }
+}
Property changes on: contrib\analyzers\smartcn\src\java\org\apache\lucene\analysis\cn\smart\hhmm\TokenGraph.java
___________________________________________________________________
Added: svn:eol-style
+ native
Index: contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java
===================================================================
--- contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (revision 831792)
+++ contrib/analyzers/smartcn/src/test/org/apache/lucene/analysis/cn/smart/TestSmartChineseAnalyzer.java (working copy)
@@ -94,6 +94,14 @@
assertAnalyzesTo(ca, sentence, result, startOffsets, endOffsets, posIncr);
}
+ /*
+ * Test analyzing an empty document
+ */
+ public void testEmpty() throws Exception {
+ Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT);
+ assertAnalyzesTo(ca, "", new String[] {});
+ }
+
public void testChineseAnalyzer() throws Exception {
Analyzer ca = new SmartChineseAnalyzer(Version.LUCENE_CURRENT, true);
String sentence = "我购买了道具和服装。";