fieldName, tokenizing
+ * the contents of text.
+ * + * This method uses {@link #createComponents(String, Reader)} to obtain an + * instance of {@link TokenStreamComponents}. It returns the sink of the + * components and stores the components internally. Subsequent calls to this + * method will reuse the previously stored components after resetting them + * through {@link TokenStreamComponents#setReader(Reader)}. + *
+ * NOTE: After calling this method, the consumer must follow the
+ * workflow described in {@link TokenStream} to properly consume its contents.
+ * See the {@link org.apache.lucene.analysis Analysis package documentation} for
+ * some examples demonstrating this.
+ *
+ * @param fieldName the name of the field the created TokenStream is used for
+ * @param text the String the streams source reads from
+ * @return TokenStream for iterating the analyzed content of reader
+ * @throws AlreadyClosedException if the Analyzer is closed.
+ * @throws IOException if an i/o error occurs (may rarely happen for Strings, but can still happen).
+ */
+ public final TokenStream tokenStream(final String fieldName, final String text) throws IOException {
+ TokenStreamComponents components = reuseStrategy.getReusableComponents(fieldName);
+ @SuppressWarnings("resource") final ReusableStringReader strReader =
+ (components == null || components.reusableStringReader == null) ?
+ new ReusableStringReader() : components.reusableStringReader;
+ strReader.setValue(text);
+ final Reader r = initReader(fieldName, strReader);
+ if (components == null) {
+ components = createComponents(fieldName, r);
+ reuseStrategy.setReusableComponents(fieldName, components);
+ } else {
+ components.setReader(r);
+ }
+ components.reusableStringReader = strReader;
+ return components.getTokenStream();
+ }
+
+ /**
* Override this if you want to add a CharFilter chain.
*
* The default implementation returns reader
@@ -208,6 +246,9 @@
* the chain. This can be the source if there are no filters.
*/
protected final TokenStream sink;
+
+ /** a cache when used with String instead of Reader */
+ transient ReusableStringReader reusableStringReader;
/**
* Creates a new {@link TokenStreamComponents} instance.
Index: lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java (working copy)
@@ -0,0 +1,61 @@
+package org.apache.lucene.analysis;
+
+import java.io.Reader;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Internal class to enable reuse of the string reader by {@link Analyzer#tokenStream(String,String)} */
+final class ReusableStringReader extends Reader {
+ private int pos = 0, size = 0;
+ private String s = null;
+
+ void setValue(String s) {
+ this.s = s;
+ this.size = s.length();
+ this.pos = 0;
+ }
+
+ @Override
+ public int read() {
+ if (pos < size) {
+ return s.charAt(pos++);
+ } else {
+ s = null;
+ return -1;
+ }
+ }
+
+ @Override
+ public int read(char[] c, int off, int len) {
+ if (pos < size) {
+ len = Math.min(len, size-pos);
+ s.getChars(pos, pos+len, c, off);
+ pos += len;
+ return len;
+ } else {
+ s = null;
+ return -1;
+ }
+ }
+
+ @Override
+ public void close() {
+ pos = size; // this prevents NPE when reading after close!
+ s = null;
+ }
+}
Index: lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java (revision 0)
+++ lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java (working copy)
Property changes on: lucene/core/src/java/org/apache/lucene/analysis/ReusableStringReader.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Date Author Id Revision HeadURL
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/java/org/apache/lucene/document/Field.java
===================================================================
--- lucene/core/src/java/org/apache/lucene/document/Field.java (revision 1500768)
+++ lucene/core/src/java/org/apache/lucene/document/Field.java (working copy)
@@ -75,7 +75,6 @@
protected TokenStream tokenStream;
private transient TokenStream internalTokenStream;
- private transient ReusableStringReader internalReader;
/**
* Field's boost
@@ -552,56 +551,12 @@
} else if (readerValue() != null) {
return analyzer.tokenStream(name(), readerValue());
} else if (stringValue() != null) {
- if (internalReader == null) {
- internalReader = new ReusableStringReader();
- }
- internalReader.setValue(stringValue());
- return analyzer.tokenStream(name(), internalReader);
+ return analyzer.tokenStream(name(), stringValue());
}
throw new IllegalArgumentException("Field must have either TokenStream, String, Reader or Number value");
}
- static final class ReusableStringReader extends Reader {
- private int pos = 0, size = 0;
- private String s = null;
-
- void setValue(String s) {
- this.s = s;
- this.size = s.length();
- this.pos = 0;
- }
-
- @Override
- public int read() {
- if (pos < size) {
- return s.charAt(pos++);
- } else {
- s = null;
- return -1;
- }
- }
-
- @Override
- public int read(char[] c, int off, int len) {
- if (pos < size) {
- len = Math.min(len, size-pos);
- s.getChars(pos, pos+len, c, off);
- pos += len;
- return len;
- } else {
- s = null;
- return -1;
- }
- }
-
- @Override
- public void close() {
- pos = size; // this prevents NPE when reading after close!
- s = null;
- }
- }
-
static final class StringTokenStream extends TokenStream {
private final CharTermAttribute termAttribute = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAttribute = addAttribute(OffsetAttribute.class);
Index: lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java (revision 0)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java (working copy)
@@ -0,0 +1,61 @@
+package org.apache.lucene.analysis;
+
+import java.nio.CharBuffer;
+
+import org.apache.lucene.util.LuceneTestCase;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class TestReusableStringReader extends LuceneTestCase {
+
+ public void test() throws Exception {
+ ReusableStringReader reader = new ReusableStringReader();
+ assertEquals(-1, reader.read());
+ assertEquals(-1, reader.read(new char[1]));
+ assertEquals(-1, reader.read(new char[2], 1, 1));
+ assertEquals(-1, reader.read(CharBuffer.wrap(new char[2])));
+
+ reader.setValue("foobar");
+ char[] buf = new char[4];
+ assertEquals(4, reader.read(buf));
+ assertEquals("foob", new String(buf));
+ assertEquals(2, reader.read(buf));
+ assertEquals("ar", new String(buf, 0, 2));
+ assertEquals(-1, reader.read(buf));
+ reader.close();
+
+ reader.setValue("foobar");
+ assertEquals(0, reader.read(buf, 1, 0));
+ assertEquals(3, reader.read(buf, 1, 3));
+ assertEquals("foo", new String(buf, 1, 3));
+ assertEquals(2, reader.read(CharBuffer.wrap(buf, 2, 2)));
+ assertEquals("ba", new String(buf, 2, 2));
+ assertEquals('r', (char) reader.read());
+ assertEquals(-1, reader.read(buf));
+ reader.close();
+
+ reader.setValue("foobar");
+ StringBuilder sb = new StringBuilder();
+ int ch;
+ while ((ch = reader.read()) != -1) {
+ sb.append((char) ch);
+ }
+ reader.close();
+ assertEquals("foobar", sb.toString());
+ }
+
+}
Index: lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java (revision 0)
+++ lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java (working copy)
Property changes on: lucene/core/src/test/org/apache/lucene/analysis/TestReusableStringReader.java
___________________________________________________________________
Added: svn:keywords
## -0,0 +1 ##
+Date Author Id Revision HeadURL
\ No newline at end of property
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/test/org/apache/lucene/document/TestField.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/document/TestField.java (revision 1500768)
+++ lucene/core/src/test/org/apache/lucene/document/TestField.java (working copy)
@@ -22,7 +22,6 @@
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
-import org.apache.lucene.document.Field.ReusableStringReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -518,39 +517,4 @@
}
}
- public void testReusableStringReader() throws Exception {
- ReusableStringReader reader = new ReusableStringReader();
- assertEquals(-1, reader.read());
- assertEquals(-1, reader.read(new char[1]));
- assertEquals(-1, reader.read(new char[2], 1, 1));
- assertEquals(-1, reader.read(CharBuffer.wrap(new char[2])));
-
- reader.setValue("foobar");
- char[] buf = new char[4];
- assertEquals(4, reader.read(buf));
- assertEquals("foob", new String(buf));
- assertEquals(2, reader.read(buf));
- assertEquals("ar", new String(buf, 0, 2));
- assertEquals(-1, reader.read(buf));
- reader.close();
-
- reader.setValue("foobar");
- assertEquals(0, reader.read(buf, 1, 0));
- assertEquals(3, reader.read(buf, 1, 3));
- assertEquals("foo", new String(buf, 1, 3));
- assertEquals(2, reader.read(CharBuffer.wrap(buf, 2, 2)));
- assertEquals("ba", new String(buf, 2, 2));
- assertEquals('r', (char) reader.read());
- assertEquals(-1, reader.read(buf));
- reader.close();
-
- reader.setValue("foobar");
- StringBuilder sb = new StringBuilder();
- int ch;
- while ((ch = reader.read()) != -1) {
- sb.append((char) ch);
- }
- reader.close();
- assertEquals("foobar", sb.toString());
- }
}