* <fieldType name="text_icu_custom" class="solr.TextField" positionIncrementGap="100"> * <analyzer> - * <tokenizer class="solr.ICUTokenizerFactory" + * <tokenizer class="solr.ICUTokenizerFactory" cjkAsWords="true" * rulefiles="Latn:my.Latin.rules.rbbi,Cyrl:my.Cyrillic.rules.rbbi"/> * </analyzer> * </fieldType>@@ -79,6 +79,7 @@ static final String RULEFILES = "rulefiles"; private final Map
- * This breaks Lao text into syllables according to: - * Syllabification of Lao Script for Line Breaking - * Phonpasit Phissamay, Valaxay Dalolay, Chitaphone Chanhsililath, Oulaiphone Silimasak, - * Sarmad Hussain, Nadir Durrani, Science Technology and Environment Agency, CRULP. - *
- * Most work is accomplished with RBBI rules, however some additional special logic is needed - * that cannot be coded in a grammar, and this is implemented here. - *
- * For example, what appears to be a final consonant might instead be part of the next syllable. - * Rules match in a greedy fashion, leaving an illegal sequence that matches no rules. - *
- * Take for instance the text ກວ່າດອກ - * The first rule greedily matches ກວ່າດ, but then ອກ is encountered, which is illegal. - * What LaoBreakIterator does, according to the paper: - *
- * Finally, LaoBreakIterator also takes care of the second concern mentioned in the paper.
- * This is the issue of combining marks being in the wrong order (typos).
- * @lucene.experimental
- */
-public class LaoBreakIterator extends BreakIterator {
- RuleBasedBreakIterator rules;
- CharArrayIterator text;
-
- CharArrayIterator working = new CharArrayIterator();
- int workingOffset = 0;
-
- CharArrayIterator verifyText = new CharArrayIterator();
- RuleBasedBreakIterator verify;
-
- private static final UnicodeSet laoSet;
- static {
- laoSet = new UnicodeSet("[:Lao:]");
- laoSet.compact();
- laoSet.freeze();
- }
-
- /**
- * Creates a new iterator, performing the backtracking verification
- * across the provided rules.
- */
- public LaoBreakIterator(RuleBasedBreakIterator rules) {
- this.rules = (RuleBasedBreakIterator) rules.clone();
- this.verify = (RuleBasedBreakIterator) rules.clone();
- }
-
- @Override
- public int current() {
- int current = rules.current();
- return current == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + current;
- }
-
- @Override
- public int first() {
- working.setText(this.text.getText(), this.text.getStart(), this.text.getLength());
- rules.setText(working);
- workingOffset = 0;
- int first = rules.first();
- return first == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + first;
- }
-
- @Override
- public int following(int offset) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public CharacterIterator getText() {
- return text;
- }
-
- @Override
- public int last() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int next() {
- int current = current();
- int next = rules.next();
- if (next == BreakIterator.DONE)
- return next;
- else
- next += workingOffset;
-
- char c = working.current();
- int following = rules.next(); // lookahead
- if (following != BreakIterator.DONE) {
- following += workingOffset;
- if (rules.getRuleStatus() == 0 && laoSet.contains(c) && verifyPushBack(current, next)) {
- workingOffset = next - 1;
- working.setText(text.getText(), text.getStart() + workingOffset, text.getLength() - workingOffset);
- return next - 1;
- }
- rules.previous(); // undo the lookahead
- }
-
- return next;
- }
-
- @Override
- public int next(int n) {
- if (n < 0)
- throw new UnsupportedOperationException("Backwards traversal is unsupported");
-
- int result = current();
- while (n > 0) {
- result = next();
- --n;
- }
- return result;
- }
-
- @Override
- public int previous() {
- throw new UnsupportedOperationException("Backwards traversal is unsupported");
- }
-
- @Override
- public void setText(CharacterIterator text) {
- if (!(text instanceof CharArrayIterator))
- throw new UnsupportedOperationException("unsupported CharacterIterator");
- this.text = (CharArrayIterator) text;
- ccReorder(this.text.getText(), this.text.getStart(), this.text.getLength());
- working.setText(this.text.getText(), this.text.getStart(), this.text.getLength());
- rules.setText(working);
- workingOffset = 0;
- }
-
- @Override
- public void setText(String newText) {
- CharArrayIterator ci = new CharArrayIterator();
- ci.setText(newText.toCharArray(), 0, newText.length());
- setText(ci);
- }
-
- private boolean verifyPushBack(int current, int next) {
- int shortenedSyllable = next - current - 1;
-
- verifyText.setText(text.getText(), text.getStart() + current, shortenedSyllable);
- verify.setText(verifyText);
- if (verify.next() != shortenedSyllable || verify.getRuleStatus() == 0)
- return false;
-
-
- verifyText.setText(text.getText(), text.getStart() + next - 1, text.getLength() - next + 1);
- verify.setText(verifyText);
-
- return (verify.next() != BreakIterator.DONE && verify.getRuleStatus() != 0);
- }
-
- // TODO: only bubblesort around runs of combining marks, instead of the entire text.
- private void ccReorder(char[] text, int start, int length) {
- boolean reordered;
- do {
- int prevCC = 0;
- reordered = false;
- for (int i = start; i < start + length; i++) {
- final char c = text[i];
- final int cc = UCharacter.getCombiningClass(c);
- if (cc > 0 && cc < prevCC) {
- // swap
- text[i] = text[i - 1];
- text[i - 1] = c;
- reordered = true;
- } else {
- prevCC = cc;
- }
- }
-
- } while (reordered == true);
- }
-
- /**
- * Clone method. Creates another LaoBreakIterator with the same behavior
- * and current state as this one.
- * @return The clone.
- */
- @Override
- public LaoBreakIterator clone() {
- LaoBreakIterator other = (LaoBreakIterator) super.clone();
- other.rules = (RuleBasedBreakIterator) rules.clone();
- other.verify = (RuleBasedBreakIterator) verify.clone();
- if (text != null)
- other.text = text.clone();
- if (working != null)
- other.working = working.clone();
- if (verifyText != null)
- other.verifyText = verifyText.clone();
- return other;
- }
-}
Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java (revision 1547187)
+++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java (working copy)
@@ -59,6 +59,15 @@
private int scriptStart;
private int scriptLimit;
private int scriptCode;
+
+ private final boolean combineCJ;
+
+ /**
+ * @param combineCJ if true: Han,Hiragana,Katakana will all return as {@link UScript#JAPANESE}
+ */
+ ScriptIterator(boolean combineCJ) {
+ this.combineCJ = combineCJ;
+ }
/**
* Get the start of this script run
@@ -162,10 +171,24 @@
}
/** fast version of UScript.getScript(). Basic Latin is an array lookup */
- private static int getScript(int codepoint) {
- if (0 <= codepoint && codepoint < basicLatin.length)
+ private int getScript(int codepoint) {
+ if (0 <= codepoint && codepoint < basicLatin.length) {
return basicLatin[codepoint];
- else
- return UScript.getScript(codepoint);
+ } else {
+ int script = UScript.getScript(codepoint);
+ if (combineCJ) {
+ if (script == UScript.HAN || script == UScript.HIRAGANA || script == UScript.KATAKANA) {
+ return UScript.JAPANESE;
+ } else if (codepoint >= 0xFF10 && codepoint <= 0xFF19) {
+ // when using CJK dictionary breaking, don't let full width numbers go to it, otherwise
+ // they are treated as punctuation. we currently have no cleaner way to fix this!
+ return UScript.LATIN;
+ } else {
+ return script;
+ }
+ } else {
+ return script;
+ }
+ }
}
}
Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java (revision 1547187)
+++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java (working copy)
@@ -84,6 +84,10 @@
@Override
public void reflectWith(AttributeReflector reflector) {
- reflector.reflect(ScriptAttribute.class, "script", getName());
+ // when wordbreaking CJK, we use the 15924 code Japanese (Han+Hiragana+Katakana) to
+ // mark runs of Chinese/Japanese. our use is correct (as for chinese Han is a subset),
+ // but this is just to help prevent confusion.
+ String name = code == UScript.JAPANESE ? "Chinese/Japanese" : getName();
+ reflector.reflect(ScriptAttribute.class, "script", name);
}
}
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Default.brk
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Hebrew.brk
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Khmer.brk
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Lao.brk
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Myanmar.brk
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
===================================================================
Cannot display: file marked as a binary type.
svn:mime-type = application/octet-stream
Index: lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java
===================================================================
--- lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (revision 1547187)
+++ lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizer.java (working copy)
@@ -42,7 +42,7 @@
sb.append(whitespace);
sb.append("testing 1234");
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input));
+ ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
}
@@ -52,7 +52,7 @@
sb.append('a');
}
String input = sb.toString();
- ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input));
+ ICUTokenizer tokenizer = new ICUTokenizer(new StringReader(input), new DefaultICUTokenizerConfig(false));
char token[] = new char[4096];
Arrays.fill(token, 'a');
String expectedToken = new String(token);
@@ -69,7 +69,7 @@
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
- Tokenizer tokenizer = new ICUTokenizer(reader);
+ Tokenizer tokenizer = new ICUTokenizer(reader, new DefaultICUTokenizerConfig(false));
TokenFilter filter = new ICUNormalizer2Filter(tokenizer);
return new TokenStreamComponents(tokenizer, filter);
}
@@ -118,6 +118,7 @@
public void testLao() throws Exception {
assertAnalyzesTo(a, "ກວ່າດອກ", new String[] { "ກວ່າ", "ດອກ" });
+ assertAnalyzesTo(a, "ພາສາລາວ", new String[] { "ພາສາ", "ລາວ"}, new String[] { "