+ * http://bugs.icu-project.org/trac/ticket/5901: RBBI.getRuleStatus(), hoist to + * BreakIterator from RuleBasedBreakIterator + *
+ * DictionaryBasedBreakIterator is a subclass of RuleBasedBreakIterator, but + * doesn't actually behave as a subclass: it always returns 0 for + * getRuleStatus(): + * http://bugs.icu-project.org/trac/ticket/4730: Thai RBBI, no boundary type + * tags + */ +abstract class BreakIteratorWrapper { + protected final CharArrayIterator textIterator = new CharArrayIterator(); + protected char text[]; + protected int start; + protected int length; + + abstract int next(); + abstract int current(); + abstract int getRuleStatus(); + abstract void setText(CharacterIterator text); + + void setText(char text[], int start, int length) { + this.text = text; + this.start = start; + this.length = length; + textIterator.setText(text, start, length); + setText(textIterator); + } + + /** + * If its a DictionaryBasedBreakIterator, it doesn't return rulestatus, so + * treat it like a generic BreakIterator If its any other + * RuleBasedBreakIterator, the rule status can be used for token type. If its + * any other BreakIterator, the rulestatus method is not available, so treat + * it like a generic BreakIterator. + */ + static BreakIteratorWrapper wrap(BreakIterator breakIterator) { + if (breakIterator instanceof RuleBasedBreakIterator + && !(breakIterator instanceof DictionaryBasedBreakIterator)) + return new RBBIWrapper((RuleBasedBreakIterator) breakIterator); + else + return new BIWrapper(breakIterator); + } + + /** + * RuleBasedBreakIterator wrapper: RuleBasedBreakIterator (as long as its not + * a DictionaryBasedBreakIterator) behaves correctly. + */ + static final class RBBIWrapper extends BreakIteratorWrapper { + private final RuleBasedBreakIterator rbbi; + + RBBIWrapper(RuleBasedBreakIterator rbbi) { + this.rbbi = rbbi; + } + + @Override + int current() { + return rbbi.current(); + } + + @Override + int getRuleStatus() { + return rbbi.getRuleStatus(); + } + + @Override + int next() { + return rbbi.next(); + } + + @Override + void setText(CharacterIterator text) { + rbbi.setText(text); + } + } + + /** + * Generic BreakIterator wrapper: Either the rulestatus method is not + * available or always returns 0. Calculate a rulestatus here so it behaves + * like RuleBasedBreakIterator. + * + * Note: This is slower than RuleBasedBreakIterator. + */ + static final class BIWrapper extends BreakIteratorWrapper { + private final BreakIterator bi; + private int status; + + BIWrapper(BreakIterator bi) { + this.bi = bi; + } + + @Override + int current() { + return bi.current(); + } + + @Override + int getRuleStatus() { + return status; + } + + @Override + int next() { + int current = bi.current(); + int next = bi.next(); + status = calcStatus(current, next); + return next; + } + + private int calcStatus(int current, int next) { + if (current == BreakIterator.DONE || next == BreakIterator.DONE) + return RuleBasedBreakIterator.WORD_NONE; + + int begin = start + current; + int end = start + next; + + int codepoint; + for (int i = begin; i < end; i += UTF16.getCharCount(codepoint)) { + codepoint = UTF16.charAt(text, 0, end, begin); + + if (UCharacter.isDigit(codepoint)) + return RuleBasedBreakIterator.WORD_NUMBER; + else if (UCharacter.isLetter(codepoint)) { + // TODO: try to separately specify ideographic, kana? + // [currently all bundled as letter for this case] + return RuleBasedBreakIterator.WORD_LETTER; + } + } + + return RuleBasedBreakIterator.WORD_NONE; + } + + @Override + void setText(CharacterIterator text) { + bi.setText(text); + status = RuleBasedBreakIterator.WORD_NONE; + } + } +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\BreakIteratorWrapper.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CharArrayIterator.java (revision 0) @@ -0,0 +1,117 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.text.CharacterIterator; + +/** + * Wraps a char[] as CharacterIterator for processing with a BreakIterator + */ +final class CharArrayIterator implements CharacterIterator { + private char array[]; + private int start; + private int index; + private int length; + private int limit; + + public char [] getText() { + return array; + } + + public int getStart() { + return start; + } + + public int getLength() { + return length; + } + + /** + * Set a new region of text to be examined by this iterator + * + * @param array text buffer to examine + * @param start offset into buffer + * @param length maximum length to examine + */ + void setText(final char array[], int start, int length) { + this.array = array; + this.start = start; + this.index = start; + this.length = length; + this.limit = start + length; + } + + public char current() { + return (index == limit) ? DONE : array[index]; + } + + public char first() { + index = start; + return current(); + } + + public int getBeginIndex() { + return 0; + } + + public int getEndIndex() { + return length; + } + + public int getIndex() { + return index - start; + } + + public char last() { + index = (limit == start) ? limit : limit - 1; + return current(); + } + + public char next() { + if (++index >= limit) { + index = limit; + return DONE; + } else { + return current(); + } + } + + public char previous() { + if (--index < start) { + index = start; + return DONE; + } else { + return current(); + } + } + + public char setIndex(int position) { + if (position < getBeginIndex() || position > getEndIndex()) + throw new IllegalArgumentException("Illegal Position: " + position); + index = start + position; + return current(); + } + + @Override + public Object clone() { + CharArrayIterator clone = new CharArrayIterator(); + clone.setText(array, start, length); + clone.index = index; + return clone; + } +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\CharArrayIterator.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/CompositeBreakIterator.java (revision 0) @@ -0,0 +1,125 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.BreakIterator; + +/** + * An internal BreakIterator for multilingual text, following recommendations + * from: UAX #29: Unicode Text Segmentation. (http://unicode.org/reports/tr29/) + *
+ * See http://unicode.org/reports/tr29/#Tailoring for the motivation of this + * design. + *
+ * Text is first divided into script boundaries. The processing is then + * delegated to the appropriate break iterator for that specific script. + *
+ * This break iterator also allows you to retrieve the ISO 15924 script code + * associated with a piece of text. + *
+ * See also UAX #29, UTR #24 + */ +final class CompositeBreakIterator { + private final ICUTokenizerConfig config; + private final BreakIteratorWrapper wordBreakers[] = new BreakIteratorWrapper[UScript.CODE_LIMIT]; + + private BreakIteratorWrapper rbbi; + private final ScriptIterator scriptIterator = new ScriptIterator(); + + private char text[]; + + CompositeBreakIterator(ICUTokenizerConfig config) { + this.config = config; + } + + /** + * Retrieve the next break position. If the RBBI range is exhausted within the + * script boundary, examine the next script boundary. + * + * @return the next break position or BreakIterator.DONE + */ + int next() { + int next = rbbi.next(); + while (next == BreakIterator.DONE && scriptIterator.next()) { + rbbi = getBreakIterator(scriptIterator.getScriptCode()); + rbbi.setText(text, scriptIterator.getScriptStart(), + scriptIterator.getScriptLimit() - scriptIterator.getScriptStart()); + next = rbbi.next(); + } + return (next == BreakIterator.DONE) ? BreakIterator.DONE : next + + scriptIterator.getScriptStart(); + } + + /** + * Retrieve the current break position. + * + * @return the current break position or BreakIterator.DONE + */ + int current() { + final int current = rbbi.current(); + return (current == BreakIterator.DONE) ? BreakIterator.DONE : current + + scriptIterator.getScriptStart(); + } + + /** + * Retrieve the rule status code (token type) from the underlying break + * iterator + * + * @return rule status code (see RuleBasedBreakIterator constants) + */ + int getRuleStatus() { + return rbbi.getRuleStatus(); + } + + /** + * Retrieve the UScript script code for the current token. This code can be + * decoded with UScript into a name or ISO 15924 code. + * + * @return UScript script code for the current token. + */ + int getScriptCode() { + return scriptIterator.getScriptCode(); + } + + /** + * Set a new region of text to be examined by this iterator + * + * @param text buffer of text + * @param start offset into buffer + * @param length maximum length to examine + */ + void setText(final char text[], int start, int length) { + this.text = text; + scriptIterator.setText(text, start, length); + if (scriptIterator.next()) { + rbbi = getBreakIterator(scriptIterator.getScriptCode()); + rbbi.setText(text, scriptIterator.getScriptStart(), + scriptIterator.getScriptLimit() - scriptIterator.getScriptStart()); + } else { + rbbi = getBreakIterator(UScript.COMMON); + rbbi.setText(text, 0, 0); + } + } + + private BreakIteratorWrapper getBreakIterator(int scriptCode) { + if (wordBreakers[scriptCode] == null) + wordBreakers[scriptCode] = BreakIteratorWrapper.wrap(config.getTokenizer(scriptCode)); + return wordBreakers[scriptCode]; + } +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\CompositeBreakIterator.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/DefaultICUTokenizerConfig.java (revision 0) @@ -0,0 +1,111 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.BreakIterator; +import com.ibm.icu.text.RuleBasedBreakIterator; +import com.ibm.icu.util.ULocale; + +/** + * Default {@link TokenizerConfig} that is generally applicable + * to many languages. + *
+ * Generally tokenizes Unicode text according to UAX#29 + * ({@link BreakIterator#getWordInstance(ULocale.ROOT)}), + * but with the following tailorings: + *
+ * Words are broken across script boundaries, then segmented according to + * the BreakIterator and typing provided by the {@link ICUTokenizerConfig} + *
+ * @see ICUTokenizerConfig + */ +public final class ICUTokenizer extends Tokenizer { + private static final int IOBUFFER = 4096; + private final char buffer[] = new char[IOBUFFER]; + /** true length of text in the buffer */ + private int length = 0; + /** length in buffer that can be evaluated safely, up to a safe end point */ + private int usableLength = 0; + /** accumulated offset of previous buffers for this reader, for offsetAtt */ + private int offset = 0; + + private final CompositeBreakIterator breaker; /* tokenizes a char[] of text */ + private final ICUTokenizerConfig config; + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); + private final ScriptAttribute scriptAtt = addAttribute(ScriptAttribute.class); + + /** + * Construct a new ICUTokenizer that breaks text into words from the given + * Reader. + *+ * The default script-specific handling is used. + * + * @param input Reader containing text to tokenize. + * @see DefaultICUTokenizerConfig + */ + public ICUTokenizer(Reader input) { + this(input, new DefaultICUTokenizerConfig()); + } + + /** + * Construct a new ICUTokenizer that breaks text into words from the given + * Reader, using a tailored BreakIterator configuration. + * + * @param input Reader containing text to tokenize. + * @param config Tailored BreakIterator configuration + */ + public ICUTokenizer(Reader input, ICUTokenizerConfig config) { + super(input); + this.config = config; + breaker = new CompositeBreakIterator(config); + } + + @Override + public boolean incrementToken() throws IOException { + clearAttributes(); + if (length == 0) + refill(); + while (!incrementTokenBuffer()) { + refill(); + if (length <= 0) // no more bytes to read; + return false; + } + return true; + } + + @Override + public void reset() throws IOException { + super.reset(); + breaker.setText(buffer, 0, 0); + length = usableLength = offset = 0; + } + + @Override + public void reset(Reader input) throws IOException { + super.reset(input); + reset(); + } + + @Override + public void end() throws IOException { + final int finalOffset = (length < 0) ? offset : offset + length; + offsetAtt.setOffset(finalOffset, finalOffset); + } + + /* + * This tokenizes text based upon the longest matching rule, and because of + * this, isn't friendly to a Reader. + * + * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of + * text, the last unambiguous break point is found (in this implementation: + * white space character) Any remaining characters represent possible partial + * words, so are appended to the front of the next chunk. + * + * There is the possibility that there are no unambiguous break points within + * an entire 4kB chunk of text (binary data). So there is a maximum word limit + * of 4kB since it will not try to grow the buffer in this case. + */ + + /** + * Returns the last unambiguous break position in the text. + * + * @return position of character, or -1 if one does not exist + */ + private int findSafeEnd() { + for (int i = length - 1; i >= 0; i--) + if (UCharacter.isWhitespace(buffer[i])) + return i + 1; + return -1; + } + + /** + * Refill the buffer, accumulating the offset and setting usableLength to the + * last unambiguous break position + * + * @throws IOException + */ + private void refill() throws IOException { + offset += usableLength; + int leftover = length - usableLength; + System.arraycopy(buffer, usableLength, buffer, 0, leftover); + int requested = buffer.length - leftover; + int returned = input.read(buffer, leftover, requested); + length = returned < 0 ? leftover : returned + leftover; + if (returned < requested) /* reader has been emptied, process the rest */ + usableLength = length; + else { /* still more data to be read, find a safe-stopping place */ + usableLength = findSafeEnd(); + if (usableLength < 0) + usableLength = length; /* + * more than IOBUFFER of text without space, + * gonna possibly truncate tokens + */ + } + + breaker.setText(buffer, 0, Math.max(0, usableLength)); + } + + /* + * return true if there is a token from the buffer, or null if it is + * exhausted. + */ + private boolean incrementTokenBuffer() { + int start = breaker.current(); + if (start == BreakIterator.DONE) + return false; // BreakIterator exhausted + + // find the next set of boundaries, skipping over non-tokens (rule status 0) + int end = breaker.next(); + while (start != BreakIterator.DONE && breaker.getRuleStatus() == 0) { + start = end; + end = breaker.next(); + } + + if (start == BreakIterator.DONE) + return false; // BreakIterator exhausted + + termAtt.copyBuffer(buffer, start, end - start); + offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end)); + typeAtt.setType(config.getType(breaker.getScriptCode(), breaker.getRuleStatus())); + scriptAtt.setCode(breaker.getScriptCode()); + + return true; + } +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\ICUTokenizer.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerConfig.java (revision 0) @@ -0,0 +1,32 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import com.ibm.icu.text.BreakIterator; + +/** + * Class that allows for tailored Unicode Text Segmentation on + * a per-writing system basis. + */ +public abstract class ICUTokenizerConfig { + /** Return a breakiterator capable of processing a given script. */ + abstract BreakIterator getTokenizer(int script); + /** Return a token type value for a given script and BreakIterator + * rule status. */ + abstract String getType(int script, int ruleStatus); +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\ICUTokenizerConfig.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/LaoBreakIterator.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/LaoBreakIterator.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/LaoBreakIterator.java (revision 0) @@ -0,0 +1,225 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.text.CharacterIterator; + +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.text.BreakIterator; +import com.ibm.icu.text.RuleBasedBreakIterator; +import com.ibm.icu.text.UnicodeSet; + +/** + * Syllable iterator for Lao text. + *
+ * This breaks Lao text into syllables according to: + * Syllabification of Lao Script for Line Breaking + * Phonpasit Phissamay, Valaxay Dalolay, Chitaphone Chanhsililath, Oulaiphone Silimasak, + * Sarmad Hussain, Nadir Durrani, Science Technology and Environment Agency, CRULP. + *
+ * Most work is accomplished with RBBI rules, however some additional special logic is needed + * that cannot be coded in a grammar, and this is implemented here. + *
+ * For example, what appears to be a final consonant might instead be part of the next syllable. + * Rules match in a greedy fashion, leaving an illegal sequence that matches no rules. + *
+ * Take for instance the text ກວ່າດອກ + * The first rule greedily matches ກວ່າດ, but then ອກ is encountered, which is illegal. + * What LaoBreakIterator does, according to the paper: + *
+ * Finally, LaoBreakIterator also takes care of the second concern mentioned in the paper. + * This is the issue of combining marks being in the wrong order (typos). + */ +public class LaoBreakIterator extends BreakIterator { + RuleBasedBreakIterator rules; + CharArrayIterator text; + + CharArrayIterator working = new CharArrayIterator(); + int workingOffset = 0; + + CharArrayIterator verifyText = new CharArrayIterator(); + RuleBasedBreakIterator verify; + + private static final UnicodeSet laoSet; + static { + laoSet = new UnicodeSet("[:Lao:]"); + laoSet.compact(); + laoSet.freeze(); + } + + public LaoBreakIterator(RuleBasedBreakIterator rules) { + this.rules = (RuleBasedBreakIterator) rules.clone(); + this.verify = (RuleBasedBreakIterator) rules.clone(); + } + + @Override + public int current() { + int current = rules.current(); + return current == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + current; + } + + @Override + public int first() { + working.setText(this.text.getText(), this.text.getStart(), this.text.getLength()); + rules.setText(working); + workingOffset = 0; + int first = rules.first(); + return first == BreakIterator.DONE ? BreakIterator.DONE : workingOffset + first; + } + + @Override + public int following(int offset) { + throw new UnsupportedOperationException(); + } + + @Override + public CharacterIterator getText() { + return text; + } + + @Override + public int last() { + throw new UnsupportedOperationException(); + } + + @Override + public int next() { + int current = current(); + int next = rules.next(); + if (next == BreakIterator.DONE) + return next; + else + next += workingOffset; + + char c = working.current(); + int following = rules.next(); // lookahead + if (following != BreakIterator.DONE) { + following += workingOffset; + if (rules.getRuleStatus() == 0 && laoSet.contains(c) && verifyPushBack(current, next)) { + workingOffset = next - 1; + working.setText(text.getText(), text.getStart() + workingOffset, text.getLength() - workingOffset); + return next - 1; + } + rules.previous(); // undo the lookahead + } + + return next; + } + + @Override + public int next(int n) { + if (n < 0) + throw new UnsupportedOperationException("Backwards traversal is unsupported"); + + int result = current(); + while (n > 0) { + result = next(); + --n; + } + return result; + } + + @Override + public int previous() { + throw new UnsupportedOperationException("Backwards traversal is unsupported"); + } + + @Override + public void setText(CharacterIterator text) { + if (!(text instanceof CharArrayIterator)) + throw new UnsupportedOperationException("unsupported CharacterIterator"); + this.text = (CharArrayIterator) text; + ccReorder(this.text.getText(), this.text.getStart(), this.text.getLength()); + working.setText(this.text.getText(), this.text.getStart(), this.text.getLength()); + rules.setText(working); + workingOffset = 0; + } + + @Override + public void setText(String newText) { + CharArrayIterator ci = new CharArrayIterator(); + ci.setText(newText.toCharArray(), 0, newText.length()); + setText(ci); + } + + private boolean verifyPushBack(int current, int next) { + int shortenedSyllable = next - current - 1; + + verifyText.setText(text.getText(), text.getStart() + current, shortenedSyllable); + verify.setText(verifyText); + if (verify.next() != shortenedSyllable || verify.getRuleStatus() == 0) + return false; + + + verifyText.setText(text.getText(), text.getStart() + next - 1, text.getLength() - next + 1); + verify.setText(verifyText); + + return (verify.next() != BreakIterator.DONE && verify.getRuleStatus() != 0); + } + + // TODO: only bubblesort around runs of combining marks, instead of the entire text. + private void ccReorder(char[] text, int start, int length) { + boolean reordered; + do { + int prevCC = 0; + reordered = false; + for (int i = start; i < start + length; i++) { + final char c = text[i]; + final int cc = UCharacter.getCombiningClass(c); + if (cc > 0 && cc < prevCC) { + // swap + text[i] = text[i - 1]; + text[i - 1] = c; + reordered = true; + } else { + prevCC = cc; + } + } + + } while (reordered == true); + } + + /** + * Clone method. Creates another LaoBreakIterator with the same behavior + * and current state as this one. + * @return The clone. + */ + @Override + public Object clone() { + LaoBreakIterator other = (LaoBreakIterator) super.clone(); + other.rules = (RuleBasedBreakIterator) rules.clone(); + other.verify = (RuleBasedBreakIterator) verify.clone(); + if (text != null) + other.text = (CharArrayIterator) text.clone(); + if (working != null) + other.working = (CharArrayIterator) working.clone(); + if (verifyText != null) + other.verifyText = (CharArrayIterator) verifyText.clone(); + return other; + } +} Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\LaoBreakIterator.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package.html =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package.html (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/package.html (revision 0) @@ -0,0 +1,22 @@ + + + +
+Tokenizer that breaks text into words with the Unicode Text Segmentation algorithm. + + \ No newline at end of file Property changes on: contrib\icu\src\java\org\apache\lucene\analysis\icu\segmentation\package.html ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java =================================================================== --- contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java (revision 0) +++ contrib/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ScriptIterator.java (revision 0) @@ -0,0 +1,169 @@ +package org.apache.lucene.analysis.icu.segmentation; + +/** + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +import com.ibm.icu.lang.UCharacter; +import com.ibm.icu.lang.UScript; +import com.ibm.icu.text.UTF16; + +/** + * An iterator that locates ISO 15924 script boundaries in text. + *+ * This is not the same as simply looking at the Unicode block, or even the + * Script property. Some characters are 'common' across multiple scripts, and + * some 'inherit' the script value of text surrounding them. + *
+ * This is similar to ICU (internal-only) UScriptRun, with the following + * differences: + *