+ * The new {@link TokenCharFunction} API uses int codepoints
+ * instead of char primitives to represent a character. This API
+ * change yields full support for supplementary characters and backwards
+ * compatibility via Lucenes {@link Version} enum.
+ *
+ * To preserve full backwards compatibility {@link CharTokenizer} uses a + * special {@link TokenCharFunction} calling the "old" char-based + * {@link CharTokenizer#isTokenChar(char)} and + * {@link CharTokenizer#normalize(char)} methods by default. + *
+ * + */ + public static abstract class TokenCharFunction { + + private final Version matchVersion; + + /** + * Creates a new {@link TokenCharFunction}. + * + * @param matchVersion + * the {@link Version} which should be used by the tokenizer to + * build the codepoints from its input. See {@link #getVersion()} + * for details. + */ + protected TokenCharFunction(Version matchVersion) { + this.matchVersion = matchVersion; + } + + /** + * Returnstrue if a codepoint should be included in a token.
+ * The tokenizer generates as tokens adjacent sequences of characters which
+ * satisfy this predicate. Characters for which this is false
+ * are used to define token boundaries and are not included in tokens.
+ *
+ * @param codepoint
+ * the codepoint
+ * @return true if and only if the codepoint should be included
+ * a token, otherwise false
+ * */
+ public abstract boolean isTokenCodePoint(int codepoint);
+
+ /**
+ * Called on each token codepoint to normalize it before it is added to the
+ * token. The default implementation simply returns the given codepoint.
+ * Subclasses may use this to, e.g., lowercase tokens.
+ *
+ * @param codepoint
+ * the codepoint
+ * @return a normalized codepoint
+ */
+ public int normalize(final int codepoint) {
+ return codepoint;
+ }
+
+ /**
+ * Returns the {@link Version} which should be used by the tokenizer to
+ * build the codepoints from its input.
+ *
+ * For versions < 3.1 {@link CharTokenizer} will create codepoints from a
+ * single char. All other version are aware of Unicode 4.0
+ * supplementary characters and will therefore combine high/low surrogate
+ * pairs into a single codepoint. To force {@link CharTokenizer} to yield
+ * the old behavior return {@link Version#LUCENE_30} or lower.
+ *
char[]
+ * of the given bufferSize.
+ *
+ * @param bufferSize
+ * the internal char buffer size, must be >= 2
+ * @return a new {@link CharacterBuffer} instance.
+ */
+ public CharacterBuffer newCharacterBuffer(final int bufferSize) {
+ if(bufferSize < 2)
+ throw new IllegalArgumentException("buffersize must be >= 2");
+ return new CharacterBuffer(new char[bufferSize], 0, 0);
+ }
+ /**
+ * Fills the {@link CharacterBuffer} with characters read from the given
+ * reader {@link Reader}. This method tries to read as many characters into
+ * the {@link CharacterBuffer} as possible, each call to fill will start
+ * filling the buffer from offset 0 up to the length of the size
+ * of the internal character array.
+ * + * Depending on the {@link Version} passed to + * {@link CharacterUtils#getInstance(Version)} this method implements + * supplementary character awareness when filling the given buffer. For all + * {@link Version} > 3.0 {@link #fill(CharacterBuffer, Reader)} guarantees + * that the given {@link CharacterBuffer} will never contain a high surrogate + * character as the last element in the buffer unless it is the last available + * character in the reader. In other words, high and low surrogate pairs will + * always be preserved across buffer boarders. + *
+ * + * @param buffer + * the buffer to fill. + * @param reader + * the reader to read characters from. + * @returntrue if and only if no more characters are available
+ * in the reader, otherwise false.
+ * @throws IOException
+ * if the reader throws an {@link IOException}.
+ */
+ public abstract boolean fill(CharacterBuffer buffer, Reader reader) throws IOException;
+
private static final class Java5CharacterUtils extends CharacterUtils {
Java5CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -124,12 +169,32 @@
return Character.codePointAt(chars, offset, limit);
}
-
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ final char[] charBuffer = buffer.buffer;
+ buffer.offset = 0;
+ charBuffer[0] = buffer.lastTrailingHighSurrogate;
+ final int offset = buffer.lastTrailingHighSurrogate == 0 ? 0 : 1;
+ buffer.lastTrailingHighSurrogate = 0;
+ final int read = reader.read(charBuffer, offset, charBuffer.length
+ - offset);
+ if (read == -1) {
+ buffer.length = offset;
+ return offset != 0;
+ }
+ buffer.length = read + offset;
+ // special case if the read returns 0 and the lastTrailingHighSurrogate was set
+ if (buffer.length > 1
+ && Character.isHighSurrogate(charBuffer[buffer.length - 1])) {
+ buffer.lastTrailingHighSurrogate = charBuffer[--buffer.length];
+ }
+ return true;
+ }
}
private static final class Java4CharacterUtils extends CharacterUtils {
Java4CharacterUtils() {
- };
+ }
@Override
public final int codePointAt(final char[] chars, final int offset) {
@@ -148,6 +213,61 @@
return chars[offset];
}
+ @Override
+ public boolean fill(final CharacterBuffer buffer, final Reader reader) throws IOException {
+ buffer.offset = 0;
+ final int read = reader.read(buffer.buffer);
+ if(read == -1)
+ return false;
+ buffer.length = read;
+ return true;
+ }
+
}
+
+ /**
+ * A simple IO buffer to use with
+ * {@link CharacterUtils#fill(CharacterBuffer, Reader)}.
+ */
+ public static final class CharacterBuffer {
+ private final char[] buffer;
+ private int offset;
+ private int length;
+ private char lastTrailingHighSurrogate = 0;
+
+ CharacterBuffer(char[] buffer, int offset, int length) {
+ this.buffer = buffer;
+ this.offset = offset;
+ this.length = length;
+ }
+
+ /**
+ * Returns the internal buffer
+ *
+ * @return the buffer
+ */
+ public char[] getBuffer() {
+ return buffer;
+ }
+
+ /**
+ * Returns the data offset in the internal buffer.
+ *
+ * @return the offset
+ */
+ public int getOffset() {
+ return offset;
+ }
+
+ /**
+ * Return the length of the data in the internal buffer starting at
+ * {@link #getOffset()}
+ *
+ * @return the length
+ */
+ public int getLength() {
+ return length;
+ }
+ }
}
Index: src/test/org/apache/lucene/util/TestCharacterUtils.java
===================================================================
--- src/test/org/apache/lucene/util/TestCharacterUtils.java (revision 0)
+++ src/test/org/apache/lucene/util/TestCharacterUtils.java (revision 0)
@@ -0,0 +1,198 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.util.CharacterUtils.CharacterBuffer;
+import org.junit.Test;
+
+/**
+ * TestCase for the {@link CharacterUtils} class.
+ */
+public class TestCharacterUtils {
+
+ @Test
+ public void testCodePointAtCharArrayInt() {
+ CharacterUtils java4 = CharacterUtils.getInstance(Version.LUCENE_30);
+ char[] cpAt3 = "Abc\ud801\udc1c".toCharArray();
+ char[] highSurrogateAt3 = "Abc\ud801".toCharArray();
+ assertEquals((int) 'A', java4.codePointAt(cpAt3, 0));
+ assertEquals((int) '\ud801', java4.codePointAt(cpAt3, 3));
+ assertEquals((int) '\ud801', java4.codePointAt(highSurrogateAt3, 3));
+ try {
+ java4.codePointAt(highSurrogateAt3, 4);
+ fail("array index out of bounds");
+ } catch (ArrayIndexOutOfBoundsException e) {
+ }
+
+ CharacterUtils java5 = CharacterUtils.getInstance(Version.LUCENE_31);
+ assertEquals((int) 'A', java5.codePointAt(cpAt3, 0));
+ assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
+ cpAt3, 3));
+ assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3));
+ try {
+ java5.codePointAt(highSurrogateAt3, 4);
+ fail("array index out of bounds");
+ } catch (ArrayIndexOutOfBoundsException e) {
+ }
+ }
+
+ @Test
+ public void testCodePointAtCharSequenceInt() {
+ CharacterUtils java4 = CharacterUtils.getInstance(Version.LUCENE_30);
+ String cpAt3 = "Abc\ud801\udc1c";
+ String highSurrogateAt3 = "Abc\ud801";
+ assertEquals((int) 'A', java4.codePointAt(cpAt3, 0));
+ assertEquals((int) '\ud801', java4.codePointAt(cpAt3, 3));
+ assertEquals((int) '\ud801', java4.codePointAt(highSurrogateAt3, 3));
+ try {
+ java4.codePointAt(highSurrogateAt3, 4);
+ fail("string index out of bounds");
+ } catch (StringIndexOutOfBoundsException e) {
+ }
+
+ CharacterUtils java5 = CharacterUtils.getInstance(Version.LUCENE_31);
+ assertEquals((int) 'A', java5.codePointAt(cpAt3, 0));
+ assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
+ cpAt3, 3));
+ assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3));
+ try {
+ java5.codePointAt(highSurrogateAt3, 4);
+ fail("string index out of bounds");
+ } catch (StringIndexOutOfBoundsException e) {
+ }
+
+ }
+
+ @Test
+ public void testCodePointAtCharArrayIntInt() {
+ CharacterUtils java4 = CharacterUtils.getInstance(Version.LUCENE_30);
+ char[] cpAt3 = "Abc\ud801\udc1c".toCharArray();
+ char[] highSurrogateAt3 = "Abc\ud801".toCharArray();
+ assertEquals((int) 'A', java4.codePointAt(cpAt3, 0, 2));
+ assertEquals((int) '\ud801', java4.codePointAt(cpAt3, 3, 5));
+ assertEquals((int) '\ud801', java4.codePointAt(highSurrogateAt3, 3, 4));
+
+ CharacterUtils java5 = CharacterUtils.getInstance(Version.LUCENE_31);
+ assertEquals((int) 'A', java5.codePointAt(cpAt3, 0, 2));
+ assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
+ cpAt3, 3, 5));
+ assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3, 4));
+
+ }
+
+ @Test
+ public void testNewCharacterBuffer() {
+ Version[] versions = new Version[] { Version.LUCENE_30, Version.LUCENE_31 };
+ for (Version version : versions) {
+ CharacterUtils charUtils = CharacterUtils.getInstance(version);
+ CharacterBuffer newCharacterBuffer = charUtils.newCharacterBuffer(1024);
+ assertEquals(1024, newCharacterBuffer.getBuffer().length);
+ assertEquals(0, newCharacterBuffer.getOffset());
+ assertEquals(0, newCharacterBuffer.getLength());
+
+ newCharacterBuffer = charUtils.newCharacterBuffer(2);
+ assertEquals(2, newCharacterBuffer.getBuffer().length);
+ assertEquals(0, newCharacterBuffer.getOffset());
+ assertEquals(0, newCharacterBuffer.getLength());
+
+ try {
+ newCharacterBuffer = charUtils.newCharacterBuffer(1);
+ fail("length must be >= 2");
+ } catch (IllegalArgumentException e) {
+ }
+ }
+ }
+
+ @Test
+ public void testFillNoHighSurrogate() throws IOException {
+ Version[] versions = new Version[] { Version.LUCENE_30, Version.LUCENE_31 };
+ for (Version version : versions) {
+ CharacterUtils instance = CharacterUtils.getInstance(version);
+ Reader reader = new StringReader("helloworld");
+ CharacterBuffer buffer = instance.newCharacterBuffer(6);
+ assertTrue(instance.fill(buffer,reader));
+ assertEquals(0, buffer.getOffset());
+ assertEquals(6, buffer.getLength());
+ assertEquals("hellow", new String(buffer.getBuffer()));
+ assertTrue(instance.fill(buffer,reader));
+ assertEquals(4, buffer.getLength());
+ assertEquals(0, buffer.getOffset());
+
+ assertEquals("orld", new String(buffer.getBuffer(), buffer.getOffset(),
+ buffer.getLength()));
+ assertFalse(instance.fill(buffer,reader));
+ }
+ }
+
+ @Test
+ public void testFillJava15() throws IOException {
+ String input = "1234\ud801\udc1c789123\ud801\ud801\udc1c\ud801";
+ CharacterUtils instance = CharacterUtils.getInstance(Version.LUCENE_31);
+ Reader reader = new StringReader(input);
+ CharacterBuffer buffer = instance.newCharacterBuffer(5);
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(4, buffer.getLength());
+ assertEquals("1234", new String(buffer.getBuffer(), buffer.getOffset(),
+ buffer.getLength()));
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(5, buffer.getLength());
+ assertEquals("\ud801\udc1c789", new String(buffer.getBuffer()));
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(4, buffer.getLength());
+ assertEquals("123\ud801", new String(buffer.getBuffer(),
+ buffer.getOffset(), buffer.getLength()));
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(2, buffer.getLength());
+ assertEquals("\ud801\udc1c", new String(buffer.getBuffer(), buffer
+ .getOffset(), buffer.getLength()));
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(1, buffer.getLength());
+ assertEquals("\ud801", new String(buffer.getBuffer(), buffer
+ .getOffset(), buffer.getLength()));
+ assertFalse(instance.fill(buffer, reader));
+ }
+
+ @Test
+ public void testFillJava14() throws IOException {
+ String input = "1234\ud801\udc1c789123\ud801\ud801\udc1c\ud801";
+ CharacterUtils instance = CharacterUtils.getInstance(Version.LUCENE_30);
+ Reader reader = new StringReader(input);
+ CharacterBuffer buffer = instance.newCharacterBuffer(5);
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(5, buffer.getLength());
+ assertEquals("1234\ud801", new String(buffer.getBuffer(), buffer
+ .getOffset(), buffer.getLength()));
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(5, buffer.getLength());
+ assertEquals("\udc1c7891", new String(buffer.getBuffer()));
+ buffer = instance.newCharacterBuffer(6);
+ assertTrue(instance.fill(buffer, reader));
+ assertEquals(6, buffer.getLength());
+ assertEquals("23\ud801\ud801\udc1c\ud801", new String(buffer.getBuffer(), buffer
+ .getOffset(), buffer.getLength()));
+ assertFalse(instance.fill(buffer, reader));
+
+ }
+
+}
Property changes on: src/test/org/apache/lucene/util/TestCharacterUtils.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native