Index: src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 533454) +++ src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -19,10 +19,13 @@ import junit.framework.TestCase; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceTokenizer; import org.apache.lucene.document.*; +import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.RAMDirectory; @@ -124,4 +127,45 @@ assertEquals(0, termPositions.nextPosition()); assertEquals(502, termPositions.nextPosition()); } + + public void testPreAnalyzedField() throws IOException { + Similarity similarity = Similarity.getDefault(); + DocumentWriter writer = new DocumentWriter(dir, new SimpleAnalyzer(), similarity, 50); + Document doc = new Document(); + + doc.add(new Field("preanalyzed", new TokenStream() { + private String[] tokens = new String[] {"term1", "term2", "term3", "term2"}; + private int index = 0; + + public Token next() throws IOException { + if (index == tokens.length) { + return null; + } else { + return new Token(tokens[index++], 0, 0); + } + } + + }, Field.Index.TOKENIZED, TermVector.NO)); + + String segName = "test"; + writer.addDocument(segName, doc); + SegmentReader reader = SegmentReader.get(new SegmentInfo(segName, 1, dir)); + + TermPositions termPositions = reader.termPositions(new Term("preanalyzed", "term1")); + assertTrue(termPositions.next()); + assertEquals(1, termPositions.freq()); + assertEquals(0, termPositions.nextPosition()); + + termPositions.seek(new Term("preanalyzed", "term2")); + assertTrue(termPositions.next()); + assertEquals(2, termPositions.freq()); + assertEquals(1, termPositions.nextPosition()); + assertEquals(3, termPositions.nextPosition()); + + termPositions.seek(new Term("preanalyzed", "term3")); + assertTrue(termPositions.next()); + assertEquals(1, termPositions.freq()); + assertEquals(2, termPositions.nextPosition()); + + } } Index: src/java/org/apache/lucene/analysis/TokenStream.java =================================================================== --- src/java/org/apache/lucene/analysis/TokenStream.java (revision 533454) +++ src/java/org/apache/lucene/analysis/TokenStream.java (working copy) @@ -35,6 +35,15 @@ /** Returns the next token in the stream, or null at EOS. */ public abstract Token next() throws IOException; + /** Resets this stream to the beginning. This is an + * optional operation, so subclasses may or may not + * implement this method. Reset() is not needed for + * the standard indexing process. However, if the Tokens + * of a TokenStream are intended to be consumed more than + * once, it is neccessary to implement reset(). + */ + public void reset() throws IOException {} + /** Releases resources associated with this stream. */ public void close() throws IOException {} } Index: src/java/org/apache/lucene/index/FieldsReader.java =================================================================== --- src/java/org/apache/lucene/index/FieldsReader.java (revision 533454) +++ src/java/org/apache/lucene/index/FieldsReader.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -331,11 +332,9 @@ return localFieldsStream; } - /** - * The value of the field in Binary, or null. If null, the Reader or - * String value is used. Exactly one of stringValue(), readerValue() and - * binaryValue() must be set. - */ + /** The value of the field in Binary, or null. If null, the Reader value, + * String value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public byte[] binaryValue() { ensureOpen(); if (fieldsData == null) { @@ -358,21 +357,26 @@ return fieldsData instanceof byte[] ? (byte[]) fieldsData : null; } - /** - * The value of the field as a Reader, or null. If null, the String value - * or binary value is used. Exactly one of stringValue(), readerValue(), - * and binaryValue() must be set. - */ + /** The value of the field as a Reader, or null. If null, the String value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public Reader readerValue() { ensureOpen(); return fieldsData instanceof Reader ? (Reader) fieldsData : null; } - /** - * The value of the field as a String, or null. If null, the Reader value - * or binary value is used. Exactly one of stringValue(), readerValue(), and - * binaryValue() must be set. - */ + /** The value of the field as a TokesStream, or null. If null, the Reader value, + * String value, or binary value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public TokenStream tokenStreamValue() { + ensureOpen(); + return fieldsData instanceof TokenStream ? (TokenStream) fieldsData : null; + } + + + /** The value of the field as a String, or null. If null, the Reader value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public String stringValue() { ensureOpen(); if (fieldsData == null) { @@ -462,6 +466,11 @@ public byte[] binaryValue() { return (byte[]) this.fieldsData; } + + public TokenStream tokenStreamValue() { + // not needed for merge + return null; + } public FieldForMerge(Object value, FieldInfo fi, boolean binary, boolean compressed, boolean tokenize) { this.isStored = true; Index: src/java/org/apache/lucene/index/DocumentWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentWriter.java (revision 533454) +++ src/java/org/apache/lucene/index/DocumentWriter.java (working copy) @@ -162,18 +162,28 @@ offset += stringValue.length(); length++; } else - { - Reader reader; // find or make Reader - if (field.readerValue() != null) - reader = field.readerValue(); - else if (field.stringValue() != null) - reader = new StringReader(field.stringValue()); - else - throw new IllegalArgumentException - ("field must have either String or Reader value"); - - // Tokenize field and add to postingTable - TokenStream stream = analyzer.tokenStream(fieldName, reader); + { // tokenized field + TokenStream stream = field.tokenStreamValue(); + + // the field does not have a TokenStream, + // so we have to obtain one from the analyzer + if (stream == null) { + Reader reader; // find or make Reader + if (field.readerValue() != null) + reader = field.readerValue(); + else if (field.stringValue() != null) + reader = new StringReader(field.stringValue()); + else + throw new IllegalArgumentException + ("field must have either String or Reader value"); + + // Tokenize field and add to postingTable + stream = analyzer.tokenStream(fieldName, reader); + } + + // reset the TokenStream to the first token + stream.reset(); + try { Token lastToken = null; for (Token t = stream.next(); t != null; t = stream.next()) { Index: src/java/org/apache/lucene/document/Field.java =================================================================== --- src/java/org/apache/lucene/document/Field.java (revision 533454) +++ src/java/org/apache/lucene/document/Field.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.IndexWriter; // for javadoc import org.apache.lucene.util.Parameter; @@ -134,21 +135,26 @@ } - /** The value of the field as a String, or null. If null, the Reader value - * or binary value is used. Exactly one of stringValue(), readerValue(), and - * binaryValue() must be set. */ + /** The value of the field as a String, or null. If null, the Reader value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public String stringValue() { return fieldsData instanceof String ? (String)fieldsData : null; } - /** The value of the field as a Reader, or null. If null, the String value - * or binary value is used. Exactly one of stringValue(), readerValue(), - * and binaryValue() must be set. */ + /** The value of the field as a Reader, or null. If null, the String value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public Reader readerValue() { return fieldsData instanceof Reader ? (Reader)fieldsData : null; } - /** The value of the field in Binary, or null. If null, the Reader or - * String value is used. Exactly one of stringValue(), readerValue() and - * binaryValue() must be set. */ + /** The value of the field in Binary, or null. If null, the Reader value, + * String value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ public byte[] binaryValue() { return fieldsData instanceof byte[] ? (byte[])fieldsData : null; } + /** The value of the field as a TokesStream, or null. If null, the Reader value, + * String value, or binary value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public TokenStream tokenStreamValue() { return fieldsData instanceof TokenStream ? (TokenStream)fieldsData : null; } + /** * Create a field by specifying its name, value and how it will * be saved in the index. Term vectors will not be stored in the index. @@ -280,6 +286,42 @@ setStoreTermVector(termVector); } + + /** + * Create a tokenized and indexed field that is not stored, optionally with + * storing term vectors. + * + * @param name The name of the field + * @param tokenStream The reader with the content + * @param index Must be tokenized or no norms. + * @param termVector Whether term vector should be stored + * @throws NullPointerException if name or reader is null + */ + public Field(String name, TokenStream tokenStream, Index index, TermVector termVector) { + + if (name == null) + throw new NullPointerException("name cannot be null"); + if (tokenStream == null) + throw new NullPointerException("tokenStream cannot be null"); + if (index != Index.TOKENIZED && index != Index.NO_NORMS) { + throw new IllegalArgumentException("index must be either TOKENIZED or NO_NORMS"); + } + + this.name = name.intern(); // field names are interned + this.fieldsData = tokenStream; + + this.isStored = false; + this.isCompressed = false; + + this.isIndexed = true; + this.isTokenized = true; + this.omitNorms = index == Index.NO_NORMS; + + this.isBinary = false; + + setStoreTermVector(termVector); + } + /** * Create a stored field with binary value. Optionally the value may be compressed. Index: src/java/org/apache/lucene/document/Fieldable.java =================================================================== --- src/java/org/apache/lucene/document/Fieldable.java (revision 533454) +++ src/java/org/apache/lucene/document/Fieldable.java (working copy) @@ -16,6 +16,8 @@ * limitations under the License. */ +import org.apache.lucene.analysis.TokenStream; + import java.io.Reader; import java.io.Serializable; @@ -60,21 +62,26 @@ */ String name(); - /** The value of the field as a String, or null. If null, the Reader value - * or binary value is used. Exactly one of stringValue(), readerValue(), and - * binaryValue() must be set. */ - String stringValue(); + /** The value of the field as a String, or null. If null, the Reader value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public String stringValue(); - /** The value of the field as a Reader, or null. If null, the String value - * or binary value is used. Exactly one of stringValue(), readerValue(), - * and binaryValue() must be set. */ - Reader readerValue(); + /** The value of the field as a Reader, or null. If null, the String value, + * binary value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public Reader readerValue(); - /** The value of the field in Binary, or null. If null, the Reader or - * String value is used. Exactly one of stringValue(), readerValue() and - * binaryValue() must be set. */ - byte[] binaryValue(); + /** The value of the field in Binary, or null. If null, the Reader value, + * String value, or TokenStream value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public byte[] binaryValue(); + /** The value of the field as a TokesStream, or null. If null, the Reader value, + * String value, or binary value is used. Exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + public TokenStream tokenStreamValue(); + /** True iff the value of the field is to be stored in the index for return with search hits. It is an error for this to be true if a field is Reader-valued. */