Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 598696) +++ CHANGES.txt (working copy) @@ -227,6 +227,10 @@ 10. LUCENE-1040: CharArraySet useful for efficiently checking set membership of text specified by char[]. (yonik) + +11. LUCENE-1058: Added CollaboratingAnalyzer to the buffered package under analysis. This can be used for Fields that use one or more tokens from + another Field. Also adds a new Field constructor that does not require a stringValue, readerValue, etc. Causes a minor change in the DocumentsWriter code that inverts the + Field. Also added a CachedAnalyzer and CachedTokenizer that work using a List of tokens, similar to CachingTokenizer (Grant Ingersoll) Optimizations Index: src/test/org/apache/lucene/analysis/buffered/TestCollaboratingAnalyzer.java =================================================================== --- src/test/org/apache/lucene/analysis/buffered/TestCollaboratingAnalyzer.java (revision 0) +++ src/test/org/apache/lucene/analysis/buffered/TestCollaboratingAnalyzer.java (revision 0) @@ -0,0 +1,132 @@ +package org.apache.lucene.analysis.buffered; + +/** + * Copyright 2004 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import junit.framework.TestCase; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.MockRAMDirectory; + +import java.io.IOException; +import java.io.StringReader; + +public class TestCollaboratingAnalyzer extends TestCase { + + + public TestCollaboratingAnalyzer(String s) { + super(s); + } + + protected void setUp() { + } + + protected void tearDown() { + + } + + public void test() throws IOException { + StringBuffer test = new StringBuffer(); + String [] goldA = {"The", "quick,", "brown", "Fox", "jumped", "over", "the", "Lazy,", "Brown", "Dogs." }; + String [] goldB = new String[goldA.length]; + for (int i = 0; i < goldA.length; i++) { + test.append(goldA[i]); + if (i < goldA.length - 1){ + test.append(' '); + } + goldB[i] = goldA[i].toLowerCase(); + } + StringReader reader = new StringReader(test.toString()); + CollaboratingAnalyzer analyzer = new SampleCollaboratingAnalyzer("a", "b"); + TokenStream tokA = analyzer.tokenStream("a", reader); + Token tok = null; + int i = 0; + while ((tok = tokA.next()) != null){ + assertTrue(tok.termText() + " is not equal to " + goldA[i], tok.termText().equals(goldA[i]) == true); + i++; + } + i = 0; + TokenStream tokB = analyzer.tokenStream("b", null); + while ((tok = tokB.next()) != null){ + assertTrue(tok.termText() + " is not equal to " + goldB[i], tok.termText().equals(goldB[i]) == true); + i++; + } + } + + + public void testFieldAnalysis() throws Exception { + MockRAMDirectory dir = new MockRAMDirectory(); + SampleCollaboratingAnalyzer sca = new SampleCollaboratingAnalyzer("source", "buffered"); + IndexWriter writer = new IndexWriter(dir, sca); + Document doc = new Document(); + Field sourceField = new Field("source", "The quick Red fox Jumped over the lazy brown Dogs", Field.Store.NO, Field.Index.TOKENIZED); + doc.add(sourceField); + Field buffField = new Field("buffered"); + doc.add(buffField); + Field nonBuffField = new Field("nonBuff", "Dogs don't like foxes", Field.Store.NO, Field.Index.TOKENIZED); + doc.add(nonBuffField); + writer.addDocument(doc); + writer.close(); + + IndexSearcher searcher = new IndexSearcher(dir); + TermQuery tq = new TermQuery(new Term("buffered", "dogs")); + Hits hits = searcher.search(tq); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.length() + " is not: " + 1, hits.length() == 1); + + tq = new TermQuery(new Term("buffered", "Dogs")); + hits = searcher.search(tq); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.length() + " is not: " + 0, hits.length() == 0); + + tq = new TermQuery(new Term("nonBuff", "Dogs")); + hits = searcher.search(tq); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.length() + " is not: " + 1, hits.length() == 1); + + } + + public void testEmptyFieldAnalysis() throws Exception { + MockRAMDirectory dir = new MockRAMDirectory(); + SampleCollaboratingAnalyzer sca = new SampleCollaboratingAnalyzer("source", "buffered"); + IndexWriter writer = new IndexWriter(dir, sca); + Document doc = new Document(); + Field sourceField = new Field("source", "The quick Red fox Jumped over the lazy brown Dogs", Field.Store.NO, Field.Index.TOKENIZED); + doc.add(sourceField); + Field buffField = new Field("buffered"); + doc.add(buffField); + Field nonBuffField = new Field("nonBuff"); + doc.add(nonBuffField); + try { + //An empty field that is not producing tokens causes an exception + writer.addDocument(doc); + assertTrue(false); + } catch (NullPointerException e) { + + } + writer.close(); + + + } +} + Property changes on: src/test/org/apache/lucene/analysis/buffered/TestCollaboratingAnalyzer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/test/org/apache/lucene/analysis/buffered/SampleCollaboratingAnalyzer.java =================================================================== --- src/test/org/apache/lucene/analysis/buffered/SampleCollaboratingAnalyzer.java (revision 0) +++ src/test/org/apache/lucene/analysis/buffered/SampleCollaboratingAnalyzer.java (revision 0) @@ -0,0 +1,31 @@ +package org.apache.lucene.analysis.buffered; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.WhitespaceTokenizer; + +import java.io.Reader; + + +/** + * + * + **/ +class SampleCollaboratingAnalyzer extends CollaboratingAnalyzer{ + SampleCollaboratingAnalyzer(String sourceField, String bufferedField) { + super(sourceField, bufferedField); + } + + protected BufferingTokenFilter createBufferingTokenFilter(TokenStream input) { + return new LowercasingBufferingTokenFilter(input); + } + + protected TokenStream createDefaultTokenStream(String fieldName, Reader reader) { + return new WhitespaceAnalyzer().tokenStream(fieldName, reader); + } + + protected Tokenizer createTokenizer(Reader reader) { + return new WhitespaceTokenizer(reader); + } +} Property changes on: src/test/org/apache/lucene/analysis/buffered/SampleCollaboratingAnalyzer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/test/org/apache/lucene/analysis/buffered/LowercasingBufferingTokenFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/buffered/LowercasingBufferingTokenFilter.java (revision 0) +++ src/test/org/apache/lucene/analysis/buffered/LowercasingBufferingTokenFilter.java (revision 0) @@ -0,0 +1,26 @@ +package org.apache.lucene.analysis.buffered; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.index.Payload; + + +/** + * + * + **/ +class LowercasingBufferingTokenFilter extends BufferingTokenFilter{ + LowercasingBufferingTokenFilter(TokenStream input) { + super(input); + } + + protected Token modifyToken(Token input) { + Token result = new Token(input.termText().toLowerCase(), input.startOffset(), input.endOffset(), input.type()); + Payload payload = input.getPayload(); + if (payload != null) { + byte[] bytes = payload.toByteArray(); + result.setPayload(new Payload(bytes, 0, bytes.length)); + } + return result; + } +} Property changes on: src/test/org/apache/lucene/analysis/buffered/LowercasingBufferingTokenFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/CachedTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/CachedTokenizer.java (revision 0) +++ src/java/org/apache/lucene/analysis/CachedTokenizer.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; + +/** + * This class can be used if the Tokens of a TokenStream + * are intended to be consumed more than once or you already have tokens that are ready pre-analyzed. + * It assumes all tokens are passed in to begin with, unlike + * the {@link org.apache.lucene.analysis.CachingTokenFilter} which caches as it goes. + * + * CachedTokenFilter implements the optional method + * {@link TokenStream#reset()}, which repositions the + * stream to the first Token. + * + */ +public class CachedTokenizer extends Tokenizer { + private List/**/ cache; + private Iterator iterator; + + public CachedTokenizer(List/**/ tokens) { + this.cache = tokens; + iterator = cache.iterator(); + } + + public Token next() throws IOException { + + if (!iterator.hasNext()) { + // the cache is exhausted, return null + return null; + } + + return (Token) iterator.next(); + } + + public void reset() throws IOException { + iterator = cache.iterator(); + } + + public List getCache() { + return cache; + } +} \ No newline at end of file Property changes on: src/java/org/apache/lucene/analysis/CachedTokenizer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/buffered/CollaboratingAnalyzer.java =================================================================== --- src/java/org/apache/lucene/analysis/buffered/CollaboratingAnalyzer.java (revision 0) +++ src/java/org/apache/lucene/analysis/buffered/CollaboratingAnalyzer.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.analysis.buffered; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; + +import java.io.Reader; + + +/** + * The CollaboratingAnalyzer coordinates the creation of Tokens to be used between two fields (source and buffered). Unlike + * the {@link org.apache.lucene.analysis.CachingTokenFilter}, the CollaboratingAnalyzer allows the bufferedField tokens + * to be different from the original tokens. + *

+ * Why is this useful? For instance, if you want to have two fields, one that lowercases all terms and the other is an + * leaves the tokens as is, with all other analysis being equal. In this case, you could setup your buffered Field to use + * the tokens that have been lowercased, while emitting the "regular" tokens for the as is case. + *

+ * Another example might be in the case of identifying proper nouns in content. Assuming you have a token filter that + * identifies proper nouns, you could use the CollaboratingAnalyzer to store the identified proper nouns in + * it's buffer to then be emitted when the proper noun field is invoked. + *

+ * ****************************** + * WARNING: In order for the CollaboratingAnalyzer to work, the sourceField must be added to the {@link org.apache.lucene.document.Document} + * before the bufferedField + * + * + * + **/ +public abstract class CollaboratingAnalyzer extends Analyzer { + protected String sourceField; + protected String bufferedField; + protected BufferingTokenFilter bufferingTF; + + + public CollaboratingAnalyzer(String sourceField, String bufferedField) { + this.sourceField = sourceField; + this.bufferedField = bufferedField; + } + + /** + * Create and return the {@link org.apache.lucene.analysis.TokenStream} to be consumed by Lucene indexing. + * + * The {@link org.apache.lucene.analysis.TokenStream} is created based on whether the + * + * @param fieldName + * @param reader + * @return + */ + public TokenStream tokenStream(String fieldName, Reader reader) { + TokenStream result = null; + if (fieldName.equals(sourceField)){ + bufferingTF = createBufferingTokenFilter(createTokenizer(reader)); + result = bufferingTF; + } else if (fieldName.equals(bufferedField)){ + result = new BufferedTokenizer(bufferingTF.getBuffer()); + } else { + result = createDefaultTokenStream(fieldName, reader); + } + return result; + + } + + /** + * Create the TokenStream to be used when the input Field is not the {@link #getSourceField()} or the {@link #getBufferedField()} + * @param fieldName The field name to use for the TokenStream + * @param reader The unmodified input {@link java.io.Reader} from {@link #tokenStream(String, java.io.Reader)} + * @return A new {@link TokenStream} + */ + protected abstract TokenStream createDefaultTokenStream(String fieldName, Reader reader); + + /** + * Create the {@link Tokenizer} to be used on the sourceField + * @param reader + * @return + */ + protected abstract Tokenizer createTokenizer(Reader reader); + + /** + * Implement your {@link org.apache.lucene.analysis.buffered.BufferingTokenFilter} as you see fit (i.e with as + * many upstream filters as desired) + * @param input The Analyzer will pass in the original TokenStream created by {@link #createTokenizer(java.io.Reader)}. + * @return A {@link BufferingTokenFilter} implemented to modify the buffered token streams as appropriate + */ + protected abstract BufferingTokenFilter createBufferingTokenFilter(TokenStream input); + + + public String getBufferedField() { + return bufferedField; + } + + public String getSourceField() { + return sourceField; + } +} Property changes on: src/java/org/apache/lucene/analysis/buffered/CollaboratingAnalyzer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/buffered/BufferedTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/buffered/BufferedTokenizer.java (revision 0) +++ src/java/org/apache/lucene/analysis/buffered/BufferedTokenizer.java (revision 0) @@ -0,0 +1,41 @@ +package org.apache.lucene.analysis.buffered; + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.Tokenizer; + +import java.io.IOException; +import java.io.Reader; +import java.util.Iterator; +import java.util.List; + + +/** + * The BufferedTokenizer returns tokens from the cache. + * + * @see org.apache.lucene.analysis.CachingTokenFilter + * + **/ +class BufferedTokenizer extends Tokenizer { + private List/**/ cache; + private Iterator/**/ iterator; + + public BufferedTokenizer(List cache) { + this.cache = cache; + iterator = cache.iterator(); + } + + public Token next() throws IOException { + if (!iterator.hasNext()) { + // the cache is exhausted, return null + return null; + } + + return (Token) iterator.next(); + } + + + protected void reset(Reader input) throws IOException { + iterator = cache.iterator(); + } + +} Property changes on: src/java/org/apache/lucene/analysis/buffered/BufferedTokenizer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/buffered/BufferingTokenFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/buffered/BufferingTokenFilter.java (revision 0) +++ src/java/org/apache/lucene/analysis/buffered/BufferingTokenFilter.java (revision 0) @@ -0,0 +1,48 @@ +package org.apache.lucene.analysis.buffered; + +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; + +import java.util.List; +import java.util.ArrayList; +import java.io.IOException; + + +/** + * + * + **/ +public abstract class BufferingTokenFilter extends TokenFilter { + + protected List/**/ buffer = new ArrayList/**/(); + + public BufferingTokenFilter(TokenStream input) { + super(input); + } + + public Token next() throws IOException { + Token result = input.next(); + if (result != null) { + Token tokenToBuffer = modifyToken(result); + if (tokenToBuffer != null) { + buffer.add(tokenToBuffer); + } + } + return result; + } + + /** + * Modify the input {@link org.apache.lucene.analysis.Token} as appropriate to be stored in {@link #getBuffer()} + *

+ * The returned Token should be a new instance of the Token. + * @param input The input {@link org.apache.lucene.analysis.Token} to modify + * @return The {@link org.apache.lucene.analysis.Token} to be buffered. Return null if the token should not be buffered. + */ + protected abstract Token modifyToken(Token input); + + + public List getBuffer() { + return buffer; + } +} Property changes on: src/java/org/apache/lucene/analysis/buffered/BufferingTokenFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/buffered/package.html =================================================================== --- src/java/org/apache/lucene/analysis/buffered/package.html (revision 0) +++ src/java/org/apache/lucene/analysis/buffered/package.html (revision 0) @@ -0,0 +1,18 @@ + + + org.apache.lucene.analysis.buffered + + +

The Buffered Analysis package provides definitions for creating Analyzers that are responsible for creating their own Tokens by + buffering them from other TokenStreams. Can be used in conjunction with a Field that has no value (binaryValue, stringValue, readerValue, tokenStreamValue) +
+
 
+ + \ No newline at end of file Property changes on: src/java/org/apache/lucene/analysis/buffered/package.html ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/Tokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/Tokenizer.java (revision 598696) +++ src/java/org/apache/lucene/analysis/Tokenizer.java (working copy) @@ -41,7 +41,9 @@ /** By default, closes the input Reader. */ public void close() throws IOException { - input.close(); + if (input != null) { + input.close(); + } } /** Reset the tokenizer to a new reader. Typically, an Index: src/java/org/apache/lucene/analysis/CachedAnalyzer.java =================================================================== --- src/java/org/apache/lucene/analysis/CachedAnalyzer.java (revision 0) +++ src/java/org/apache/lucene/analysis/CachedAnalyzer.java (revision 0) @@ -0,0 +1,25 @@ +package org.apache.lucene.analysis; + +import java.io.Reader; +import java.util.List; + + +/** + * Very simple Analyzer that wraps a {@link java.util.List} of {@link org.apache.lucene.analysis.Token}s + * and spits them back out via the {@link org.apache.lucene.analysis.CachedTokenizer}. + * + * @see org.apache.lucene.analysis.CachingTokenFilter for a {@link org.apache.lucene.analysis.TokenFilter} that caches as it processes the stream + * as opposed to this version which assumes the Tokens have already been processed. + * + **/ +public class CachedAnalyzer extends Analyzer{ + private List/**/ cache; + + public CachedAnalyzer(List/**/ cache) { + this.cache = cache; + } + + public TokenStream tokenStream(String fieldName, Reader reader) { + return new CachedTokenizer(cache); + } +} Property changes on: src/java/org/apache/lucene/analysis/CachedAnalyzer.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/analysis/Analyzer.java =================================================================== --- src/java/org/apache/lucene/analysis/Analyzer.java (revision 598696) +++ src/java/org/apache/lucene/analysis/Analyzer.java (working copy) @@ -32,10 +32,14 @@ */ public abstract class Analyzer { /** Creates a TokenStream which tokenizes all the text in the provided - Reader. Default implementation forwards to tokenStream(Reader) for - compatibility with older version. Override to allow Analyzer to choose + Reader. Override to allow Analyzer to choose strategy based on document and/or field. Must be able to handle null - field name for backward compatibility. */ + field name for backward compatibility. + * @param fieldName The name of the field being analyzed + * @param reader The Reader to use to get the content from + * @return The {@link org.apache.lucene.analysis.TokenStream} + + */ public abstract TokenStream tokenStream(String fieldName, Reader reader); /** Creates a TokenStream that is allowed to be re-used Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 598696) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -1304,10 +1304,13 @@ reader = readerValue; else { String stringValue = field.stringValue(); - if (stringValue == null) - throw new IllegalArgumentException("field must have either TokenStream, String or Reader value"); - stringReader.init(stringValue); - reader = stringReader; + if (stringValue != null){ + //throw new IllegalArgumentException("field must have either TokenStream, String or Reader value"); + stringReader.init(stringValue); + reader = stringReader; + } else{ + reader = null;//This is a special case where the Analyzer is caching the tokens + } } // Tokenize field and add to postingTable Index: src/java/org/apache/lucene/document/Field.java =================================================================== --- src/java/org/apache/lucene/document/Field.java (revision 598696) +++ src/java/org/apache/lucene/document/Field.java (working copy) @@ -18,7 +18,7 @@ */ import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.index.IndexWriter; // for javadoc +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.util.Parameter; import java.io.Reader; @@ -136,23 +136,34 @@ /** The value of the field as a String, or null. If null, the Reader value, - * binary value, or TokenStream value is used. Exactly one of stringValue(), - * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + * binary value, or TokenStream value is used. It is recommended that exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. + * However, if you are using the {@link org.apache.lucene.analysis.buffered.CollaboratingAnalyzer} or an {@link org.apache.lucene.analysis.Analyzer} + * that produces its own tokens, the value need not be set and the {@link #Field(String)} constructor can be used. Note this functionality is for experts only. + * + * */ public String stringValue() { return fieldsData instanceof String ? (String)fieldsData : null; } /** The value of the field as a Reader, or null. If null, the String value, - * binary value, or TokenStream value is used. Exactly one of stringValue(), - * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + * binary value, or TokenStream value is used. It is recommended that exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. + * * However, if you are using the {@link org.apache.lucene.analysis.buffered.CollaboratingAnalyzer} or an {@link org.apache.lucene.analysis.Analyzer} + * that produces its own tokens, the value need not be set and the {@link #Field(String)} constructor can be used. Note this functionality is for experts only. + * */ public Reader readerValue() { return fieldsData instanceof Reader ? (Reader)fieldsData : null; } /** The value of the field in Binary, or null. If null, the Reader value, - * String value, or TokenStream value is used. Exactly one of stringValue(), - * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + * String value, or TokenStream value is used. It is recommended that exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. + * * However, if you are using the {@link org.apache.lucene.analysis.buffered.CollaboratingAnalyzer} or an {@link org.apache.lucene.analysis.Analyzer} + * that produces its own tokens, the value need not be set and the {@link #Field(String)} constructor can be used. Note this functionality is for experts only.*/ public byte[] binaryValue() { return fieldsData instanceof byte[] ? (byte[])fieldsData : null; } /** The value of the field as a TokesStream, or null. If null, the Reader value, - * String value, or binary value is used. Exactly one of stringValue(), - * readerValue(), binaryValue(), and tokenStreamValue() must be set. */ + * String value, or binary value is used. It is recommended that exactly one of stringValue(), + * readerValue(), binaryValue(), and tokenStreamValue() must be set. + * * However, if you are using the {@link org.apache.lucene.analysis.buffered.CollaboratingAnalyzer} or an {@link org.apache.lucene.analysis.Analyzer} + * that produces its own tokens, the value need not be set and the {@link #Field(String)} constructor can be used. Note this functionality is for experts only.*/ public TokenStream tokenStreamValue() { return fieldsData instanceof TokenStream ? (TokenStream)fieldsData : null; } @@ -368,6 +379,45 @@ setStoreTermVector(termVector); } + /** + * Expert-only + * @see #Field(String, org.apache.lucene.document.Field.TermVector) + * + * Passes in {@link org.apache.lucene.document.Field.TermVector#NO}. + */ + public Field(String name){ + this(name, TermVector.NO); + } + + /** + * Expert Only
+ * A Field that is not based on any value, instead it is derived from the Analyzer itself. Contents are not stored. + *
+ * A {@link NullPointerException} may be thrown if no {@link org.apache.lucene.analysis.Token}s are produced by the Analyzer + * + * @param name The name of the field. Will be interned. + * @param termVector Info on storing Term Vectors. + * + * @see org.apache.lucene.analysis.buffered.CollaboratingAnalyzer + */ + public Field(String name, TermVector termVector){ + if (name == null) + throw new NullPointerException("name cannot be null"); + + this.name = name.intern(); // field names are interned + //this.fieldsData = null; //fieldsData is not set b/c this is an Analyzer based Field + + this.isStored = false; + + this.isIndexed = true; + this.isTokenized = true; + + this.isBinary = false; + + setStoreTermVector(termVector); + } + + /** * Create a stored field with binary value. Optionally the value may be compressed.