Index: lucene/src/java/org/apache/lucene/index/IndexableField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexableField.java (revision 1146155) +++ lucene/src/java/org/apache/lucene/index/IndexableField.java Mon Jul 18 00:38:31 NZST 2011 @@ -17,10 +17,12 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.NumericField; +import org.apache.lucene.util.AttributeConsumer; import org.apache.lucene.util.BytesRef; // nocommit jdocs @@ -64,6 +66,8 @@ // analyzer public TokenStream tokenStreamValue(); + public void consume(AttributeConsumer attributeConsumer) throws IOException; + // Numeric field: public boolean numeric(); public NumericField.DataType numericDataType(); Index: lucene/src/java/org/apache/lucene/analysis/TokenStream.java =================================================================== --- lucene/src/java/org/apache/lucene/analysis/TokenStream.java (revision 990822) +++ lucene/src/java/org/apache/lucene/analysis/TokenStream.java Mon Jul 18 00:38:31 NZST 2011 @@ -25,6 +25,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeConsumer; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeSource; @@ -178,5 +179,19 @@ /** Releases resources associated with this stream. */ public void close() throws IOException {} - + + public final void consume(AttributeConsumer consumer) throws IOException { + try { + consumer.setAttributeSource(this); + reset(); + while (incrementToken()) { + consumer.next(); -} + } + end(); + consumer.end(); + } finally { + close(); + } + } + +} Index: lucene/src/java/org/apache/lucene/document2/Field.java =================================================================== --- lucene/src/java/org/apache/lucene/document2/Field.java (revision 1146154) +++ lucene/src/java/org/apache/lucene/document2/Field.java Mon Jul 18 00:40:25 NZST 2011 @@ -17,11 +17,14 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.NumericField; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.ReusableStringReader; +import org.apache.lucene.util.AttributeConsumer; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; @@ -48,6 +51,8 @@ protected float boost = 1.0f; + private final ReusableStringReader stringReader = new ReusableStringReader(); + public Field(String name, FieldType type) { this.name = name; this.type = type; @@ -217,7 +222,23 @@ } this.tokenStream = tokenStream; } - + + public void consume(AttributeConsumer attributeConsumer) throws IOException { + TokenStream stream; + if (tokenStreamValue() != null) { + stream = tokenStreamValue(); + } else if (readerValue() != null) { + stream = this.type.analyzer().reusableTokenStream(name, readerValue()); + } else if (stringValue() != null) { + stringReader.init(stringValue()); + stream = this.type.analyzer().reusableTokenStream(name, stringReader); + } else { + throw new IllegalStateException("Field must have a value"); + } + + stream.consume(attributeConsumer); + } + public String name() { return name; } Index: lucene/src/java/org/apache/lucene/document2/FieldType.java =================================================================== --- lucene/src/java/org/apache/lucene/document2/FieldType.java (revision 1142831) +++ lucene/src/java/org/apache/lucene/document2/FieldType.java Mon Jul 18 00:12:25 NZST 2011 @@ -1,5 +1,7 @@ package org.apache.lucene.document2; +import org.apache.lucene.analysis.Analyzer; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -29,6 +31,7 @@ private boolean omitTermFreqsAndPositions; private boolean lazy; private boolean frozen; + private Analyzer analyzer; public FieldType(FieldType ref) { this.indexed = ref.indexed(); @@ -136,6 +139,15 @@ this.lazy = value; } + public void setAnalyzer(Analyzer analyzer) { + checkIfFrozen(); + this.analyzer = analyzer; + } + + public Analyzer analyzer() { + return this.analyzer; + } + /** Prints a Field for human consumption. */ @Override public final String toString() { Index: lucene/src/java/org/apache/lucene/util/AttributeConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/util/AttributeConsumer.java Mon Jul 18 00:31:28 NZST 2011 +++ lucene/src/java/org/apache/lucene/util/AttributeConsumer.java Mon Jul 18 00:31:28 NZST 2011 @@ -0,0 +1,28 @@ +package org.apache.lucene.util; + +import java.io.IOException; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public interface AttributeConsumer { + + void setAttributeSource(AttributeSource attributeSource); + + void next() throws IOException; + + void end(); +} Index: lucene/src/java/org/apache/lucene/index/ReusableStringReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/ReusableStringReader.java (revision 945420) +++ lucene/src/java/org/apache/lucene/index/ReusableStringReader.java Mon Jul 18 00:38:31 NZST 2011 @@ -22,11 +22,11 @@ /** Used by DocumentsWriter to implemented a StringReader * that can be reset to a new string; we use this when * tokenizing the string value from a Field. */ -final class ReusableStringReader extends Reader { +public final class ReusableStringReader extends Reader { int upto; int left; String s; - void init(String s) { + public void init(String s) { this.s = s; left = s.length(); this.upto = 0; Index: lucene/src/java/org/apache/lucene/util/AttributeSource.java =================================================================== --- lucene/src/java/org/apache/lucene/util/AttributeSource.java (revision 1096073) +++ lucene/src/java/org/apache/lucene/util/AttributeSource.java Mon Jul 18 00:46:38 NZST 2011 @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.io.IOException; import java.lang.ref.WeakReference; import java.util.Collections; import java.util.NoSuchElementException; @@ -370,6 +371,13 @@ } while (state != null); } + public void consume(AttributeConsumer consumer) throws IOException { + // TODO nocommit what to do here? + consumer.setAttributeSource(this); + consumer.next(); + consumer.end(); + } + @Override public int hashCode() { int code = 0; Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (revision 1134051) +++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Mon Jul 18 01:38:10 NZST 2011 @@ -17,12 +17,13 @@ * limitations under the License. */ -import java.io.IOException; -import java.io.Reader; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.AttributeConsumer; +import org.apache.lucene.util.AttributeSource; +import java.io.IOException; + /** * Holds state for inverting all occurrences of a single * field in the document. This class doesn't do anything @@ -102,56 +103,19 @@ fieldState.length++; fieldState.position++; } else { // tokenized field - final TokenStream stream; - final TokenStream streamValue = field.tokenStreamValue(); + AttributeConsumer attributeConsumer = new AttributeConsumer() { - if (streamValue != null) { - stream = streamValue; - } else { - // the field does not have a TokenStream, - // so we have to obtain one from the analyzer - final Reader reader; // find or make Reader - final Reader readerValue = field.readerValue(); + private OffsetAttribute offsetAttribute; + private PositionIncrementAttribute posIncrAttribute; - if (readerValue != null) { - reader = readerValue; - } else { - String stringValue = field.stringValue(); - if (stringValue == null) { - throw new IllegalArgumentException("field must have either TokenStream, String or Reader value"); + public void setAttributeSource(AttributeSource attributeSource) { + offsetAttribute = attributeSource.addAttribute(OffsetAttribute.class); + posIncrAttribute = attributeSource.addAttribute(PositionIncrementAttribute.class); + fieldState.attributeSource = attributeSource; + consumer.start(field); - } + } - parent.stringReader.init(stringValue); - reader = parent.stringReader; - } - + - // Tokenize field and add to postingTable - stream = docState.analyzer.reusableTokenStream(fieldInfo.name, reader); - } - - // reset the TokenStream to the first token - stream.reset(); - - try { - boolean hasMoreTokens = stream.incrementToken(); - - fieldState.attributeSource = stream; - - OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class); - PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class); - - consumer.start(field); - - for(;;) { - - // If we hit an exception in stream.next below - // (which is fairly common, eg if analyzer - // chokes on a given document), then it's - // non-aborting and (above) this one document - // will be marked as deleted, but still - // consume a docID - - if (!hasMoreTokens) break; - + public void next() throws IOException { final int posIncr = posIncrAttribute.getPositionIncrement(); fieldState.position += posIncr; if (fieldState.position > 0) { @@ -178,16 +142,13 @@ } fieldState.length++; fieldState.position++; - - hasMoreTokens = stream.incrementToken(); } - // trigger streams to perform end-of-stream operations - stream.end(); - + + public void end() { - fieldState.offset += offsetAttribute.endOffset(); + fieldState.offset += offsetAttribute.endOffset(); - } finally { - stream.close(); - } + } + }; + field.consume(attributeConsumer); } fieldState.offset += docState.analyzer.getOffsetGap(field); Index: lucene/src/test/org/apache/lucene/TestDemo.java =================================================================== --- lucene/src/test/org/apache/lucene/TestDemo.java (revision 1145297) +++ lucene/src/test/org/apache/lucene/TestDemo.java Mon Jul 18 01:18:36 NZST 2011 @@ -56,6 +56,7 @@ String longTerm = "longtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongtermlongterm"; String text = "This is the text to be indexed. " + longTerm; FieldType textType = new FieldType(TextField.TYPE_UNSTORED); + textType.setAnalyzer(analyzer); textType.setStored(true); doc.add(newField("fieldname", text, textType)); iwriter.addDocument(doc); Index: lucene/src/java/org/apache/lucene/document/Document.java =================================================================== --- lucene/src/java/org/apache/lucene/document/Document.java (revision 1134051) +++ lucene/src/java/org/apache/lucene/document/Document.java Mon Jul 18 01:11:15 NZST 2011 @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import java.util.*; @@ -25,6 +26,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.search.IndexSearcher; // for javadoc import org.apache.lucene.search.ScoreDoc; // for javadoc +import org.apache.lucene.util.AttributeConsumer; import org.apache.lucene.util.BytesRef; /** Documents are the unit of indexing and search. @@ -98,6 +100,10 @@ return field.tokenStreamValue(); } + public void consume(AttributeConsumer attributeConsumer) throws IOException { + throw new UnsupportedOperationException("Cannot support this"); + } + public boolean numeric() { return field instanceof NumericField; }