Index: solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java =================================================================== --- solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (revision 1159961) +++ solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java (working copy) @@ -563,8 +563,8 @@ DocList dl = ((ResultContext) rsp.getValues().get("response")).docs; Document d = req.getSearcher().doc(dl.iterator().nextDoc()); // ensure field is not lazy, only works for Non-Numeric fields currently (if you change schema behind test, this may fail) - assertFalse( ((Field) d.getField("test_hlt")).lazy() ); - assertFalse( ((Field) d.getField("title")).lazy() ); + assertFalse( ((Field) d.getField("test_hlt")).getClass().getSimpleName().equals("LazyField")); + assertFalse( ((Field) d.getField("title")).getClass().getSimpleName().equals("LazyField")); req.close(); } @@ -587,8 +587,8 @@ Document d = req.getSearcher().doc(di.nextDoc()); // ensure field is lazy System.out.println(d.getField("test_hlt").getClass()); - assertTrue( ((Field) d.getField("test_hlt")).lazy() ); - assertFalse( ((Field) d.getField("title")).lazy() ); + assertTrue( ((Field) d.getField("test_hlt")).getClass().getSimpleName().equals("LazyField")); + assertFalse( ((Field) d.getField("title")).getClass().getSimpleName().equals("LazyField")); req.close(); } Index: solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (revision 1159961) +++ solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (working copy) @@ -177,7 +177,7 @@ flags.append( (f != null && f.storeTermVectorOffsets()) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-' ); flags.append( (f != null && f.storeTermVectorPositions()) ? FieldFlag.TERM_VECTOR_POSITION.getAbbreviation() : '-' ); flags.append( (f != null && f.omitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-' ); - flags.append( (f != null && ((Field) f).lazy()) ? FieldFlag.LAZY.getAbbreviation() : '-' ); + flags.append( (f != null && f.getClass().getSimpleName().equals("LazyField")) ? FieldFlag.LAZY.getAbbreviation() : '-' ); flags.append( (f != null && f.binaryValue(null)!=null) ? FieldFlag.BINARY.getAbbreviation() : '-' ); flags.append( (false) ? FieldFlag.SORT_MISSING_FIRST.getAbbreviation() : '-' ); // SchemaField Specific flags.append( (false) ? FieldFlag.SORT_MISSING_LAST.getAbbreviation() : '-' ); // SchemaField Specific Index: lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java (revision 1159961) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/TestContribFieldsReader.java (working copy) @@ -88,7 +88,7 @@ assertTrue("doc is null and it shouldn't be", doc != null); IndexableField field = doc.getField(DocHelper.LAZY_FIELD_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("field is not lazy and it should be", ((Field) field).lazy()); + assertTrue("field is not lazy and it should be", field.getClass().getSimpleName().equals("LazyField")); String value = field.stringValue(); assertTrue("value is null and it shouldn't be", value != null); assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true); @@ -96,15 +96,15 @@ field = doc.getField(DocHelper.TEXT_FIELD_1_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == false); + assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); field = doc.getField(DocHelper.TEXT_FIELD_UTF1_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == false); + assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true); field = doc.getField(DocHelper.TEXT_FIELD_UTF2_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == true); + assertTrue("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true); field = doc.getField(DocHelper.LAZY_FIELD_BINARY_KEY); @@ -153,7 +153,7 @@ assertTrue("doc is null and it shouldn't be", doc != null); IndexableField field = doc.getField(DocHelper.LAZY_FIELD_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("field is not lazy and it should be", ((Field) field).lazy()); + assertTrue("field is not lazy and it should be", field.getClass().getSimpleName().equals("LazyField")); String value = field.stringValue(); assertTrue("value is null and it shouldn't be", value != null); assertTrue(value + " is not equal to " + DocHelper.LAZY_FIELD_TEXT, value.equals(DocHelper.LAZY_FIELD_TEXT) == true); @@ -161,18 +161,18 @@ field = doc.getField(DocHelper.TEXT_FIELD_1_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == false); + assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue()); field = doc.getField(DocHelper.TEXT_FIELD_UTF1_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == false); + assertFalse("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF1_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF1_TEXT) == true); assertTrue("calling stringValue() twice should give same reference", field.stringValue() == field.stringValue()); field = doc.getField(DocHelper.TEXT_FIELD_UTF2_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("Field is lazy and it should not be", ((Field) field).lazy() == true); + assertTrue("Field is lazy and it should not be", field.getClass().getSimpleName().equals("LazyField")); assertTrue(field.stringValue() + " is not equal to " + DocHelper.FIELD_UTF2_TEXT, field.stringValue().equals(DocHelper.FIELD_UTF2_TEXT) == true); assertTrue("calling stringValue() twice should give different references", field.stringValue() != field.stringValue()); @@ -247,7 +247,7 @@ assertTrue("doc is null and it shouldn't be", doc != null); IndexableField field = doc.getField(DocHelper.LARGE_LAZY_FIELD_KEY); assertTrue("field is null and it shouldn't be", field != null); - assertTrue("field is lazy", ((Field) field).lazy() == false); + assertFalse("field is lazy", field.getClass().getSimpleName().equals("LazyField")); String value; long start; long finish; @@ -265,7 +265,7 @@ reader = IndexReader.open(tmpDir); doc = getDocument(reader, 0, fieldSelector); field = doc.getField(DocHelper.LARGE_LAZY_FIELD_KEY); - assertTrue("field is not lazy", ((Field) field).lazy() == true); + assertTrue("field is not lazy", field.getClass().getSimpleName().equals("LazyField")); start = System.currentTimeMillis(); //On my machine this took around 50 - 70ms value = field.stringValue(); Index: lucene/src/test/org/apache/lucene/index/TestIndexableField.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexableField.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestIndexableField.java (revision 0) @@ -0,0 +1,391 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockFixedLengthPayloadFilter; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.document.BinaryField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.NumericField.DataType; +import org.apache.lucene.document.NumericField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; +import org.apache.lucene.index.values.PerDocFieldValues; +import org.apache.lucene.index.values.ValueType; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Lock; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.NoLockFactory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SingleInstanceLockFactory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.ThreadInterruptedException; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util._TestUtil; + +public class TestIndexableField extends LuceneTestCase { + + private class MyField implements IndexableField { + + private final int counter; + + public MyField(int counter) { + this.counter = counter; + } + + @Override + public String name() { + return "f" + counter; + } + + @Override + public float boost() { + return 1.0f + random.nextFloat(); + } + + @Override + public boolean stored() { + return (counter & 1) == 0 || (counter % 10) == 3; + } + + @Override + public BytesRef binaryValue(BytesRef reuse) { + if ((counter%10) == 3) { + final byte[] bytes = new byte[10]; + for(int idx=0;idx() { + @Override + public Iterator iterator() { + return new Iterator() { + int fieldUpto; + + @Override + public boolean hasNext() { + return fieldUpto < fieldCount; + } + + @Override + public IndexableField next() { + assert fieldUpto < fieldCount; + if (fieldUpto == 0) { + fieldUpto = 1; + return newField("id", ""+finalDocCount, StringField.TYPE_STORED); + } else { + return new MyField(finalBaseCount + (fieldUpto++-1)); + } + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + }; + } + }); + } + + final IndexReader r = w.getReader(); + w.close(); + + final IndexSearcher s = new IndexSearcher(r); + int counter = 0; + for(int id=0;id 0) fieldState.position += docState.analyzer == null ? 0 : docState.analyzer.getPositionIncrementGap(fieldInfo.name); - // nocommit -- this logic should be outside of - // indexer + // TODO (LUCENE-2309): this analysis logic should be + // outside of indexer -- field should simply give us + // a TokenStream, even for multi-valued fields if (!field.tokenized()) { // un-tokenized field String stringValue = field.stringValue(); Index: lucene/src/java/org/apache/lucene/index/FieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldsReader.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/index/FieldsReader.java (working copy) @@ -297,230 +297,4 @@ private void skipFieldBytes(int toRead) throws IOException { fieldsStream.seek(fieldsStream.getFilePointer() + toRead); } - - /* - private NumericField loadNumericField(FieldInfo fi, int numeric) throws IOException { - assert numeric != 0; - switch(numeric) { - case FieldsWriter.FIELD_IS_NUMERIC_INT: - return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setIntValue(fieldsStream.readInt()); - case FieldsWriter.FIELD_IS_NUMERIC_LONG: - return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setLongValue(fieldsStream.readLong()); - case FieldsWriter.FIELD_IS_NUMERIC_FLOAT: - return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setFloatValue(Float.intBitsToFloat(fieldsStream.readInt())); - case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE: - return new NumericField(fi.name, Field.Store.YES, fi.isIndexed).setDoubleValue(Double.longBitsToDouble(fieldsStream.readLong())); - default: - throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric)); - } - } - */ - - /* - private void addFieldLazy(Document doc, FieldInfo fi, boolean binary, boolean tokenize, boolean cacheResult, int numeric) throws IOException { - final AbstractField f; - if (binary) { - int toRead = fieldsStream.readVInt(); - long pointer = fieldsStream.getFilePointer(); - f = new LazyField(fi.name, Field.Store.YES, toRead, pointer, binary, cacheResult); - //Need to move the pointer ahead by toRead positions - fieldsStream.seek(pointer + toRead); - } else if (numeric != 0) { - f = loadNumericField(fi, numeric); - } else { - Field.Store store = Field.Store.YES; - Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize); - Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector); - - int length = fieldsStream.readVInt(); - long pointer = fieldsStream.getFilePointer(); - //Skip ahead of where we are by the length of what is stored - fieldsStream.seek(pointer+length); - f = new LazyField(fi.name, store, index, termVector, length, pointer, binary, cacheResult); - } - - f.setOmitNorms(fi.omitNorms); - f.setIndexOptions(fi.indexOptions); - doc.add(f); - } - - private void addField(Document doc, FieldInfo fi, boolean binary, boolean tokenize, int numeric) throws CorruptIndexException, IOException { - final AbstractField f; - - if (binary) { - int toRead = fieldsStream.readVInt(); - final byte[] b = new byte[toRead]; - fieldsStream.readBytes(b, 0, b.length); - f = new Field(fi.name, b); - } else if (numeric != 0) { - f = loadNumericField(fi, numeric); - } else { - Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize); - Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector); - f = new Field(fi.name, // name - fieldsStream.readString(), // read value - Field.Store.YES, - index, - termVector); - } - - f.setIndexOptions(fi.indexOptions); - f.setOmitNorms(fi.omitNorms); - doc.add(f); - } - - // Add the size of field as a byte[] containing the 4 bytes of the integer byte size (high order byte first; char = 2 bytes) - // Read just the size -- caller must skip the field content to continue reading fields - // Return the size in bytes or chars, depending on field type - private int addFieldSize(Document doc, FieldInfo fi, boolean binary, int numeric) throws IOException { - final int bytesize, size; - switch(numeric) { - case 0: - size = fieldsStream.readVInt(); - bytesize = binary ? size : 2*size; - break; - case FieldsWriter.FIELD_IS_NUMERIC_INT: - case FieldsWriter.FIELD_IS_NUMERIC_FLOAT: - size = bytesize = 4; - break; - case FieldsWriter.FIELD_IS_NUMERIC_LONG: - case FieldsWriter.FIELD_IS_NUMERIC_DOUBLE: - size = bytesize = 8; - break; - default: - throw new FieldReaderException("Invalid numeric type: " + Integer.toHexString(numeric)); - } - byte[] sizebytes = new byte[4]; - sizebytes[0] = (byte) (bytesize>>>24); - sizebytes[1] = (byte) (bytesize>>>16); - sizebytes[2] = (byte) (bytesize>>> 8); - sizebytes[3] = (byte) bytesize ; - doc.add(new Field(fi.name, sizebytes)); - return size; - } - */ - - /** - * A Lazy implementation of Fieldable that defers loading of fields until asked for, instead of when the Document is - * loaded. - * - private class LazyField extends AbstractField implements Fieldable { - private int toRead; - private long pointer; - private final boolean cacheResult; - - public LazyField(String name, Field.Store store, int toRead, long pointer, boolean isBinary, boolean cacheResult) { - super(name, store, Field.Index.NO, Field.TermVector.NO); - this.toRead = toRead; - this.pointer = pointer; - this.isBinary = isBinary; - this.cacheResult = cacheResult; - if (isBinary) - binaryLength = toRead; - lazy = true; - } - - public LazyField(String name, Field.Store store, Field.Index index, Field.TermVector termVector, int toRead, long pointer, boolean isBinary, boolean cacheResult) { - super(name, store, index, termVector); - this.toRead = toRead; - this.pointer = pointer; - this.isBinary = isBinary; - this.cacheResult = cacheResult; - if (isBinary) - binaryLength = toRead; - lazy = true; - } - - private IndexInput getFieldStream() { - IndexInput localFieldsStream = fieldsStreamTL.get(); - if (localFieldsStream == null) { - localFieldsStream = (IndexInput) cloneableFieldsStream.clone(); - fieldsStreamTL.set(localFieldsStream); - } - return localFieldsStream; - } - - ** The value of the field as a Reader, or null. If null, the String value, - * binary value, or TokenStream value is used. Exactly one of stringValue(), - * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. * - public Reader readerValue() { - ensureOpen(); - return null; - } - - /** The value of the field as a TokenStream, or null. If null, the Reader value, - * String value, or binary value is used. Exactly one of stringValue(), - * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. * - public TokenStream tokenStreamValue() { - ensureOpen(); - return null; - } - - /** The value of the field as a String, or null. If null, the Reader value, - * binary value, or TokenStream value is used. Exactly one of stringValue(), - * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. * - public String stringValue() { - ensureOpen(); - if (isBinary) - return null; - else { - if (fieldsData == null) { - String result = null; - IndexInput localFieldsStream = getFieldStream(); - try { - localFieldsStream.seek(pointer); - byte[] bytes = new byte[toRead]; - localFieldsStream.readBytes(bytes, 0, toRead); - result = new String(bytes, "UTF-8"); - } catch (IOException e) { - throw new FieldReaderException(e); - } - if (cacheResult == true){ - fieldsData = result; - } - return result; - } else { - return (String) fieldsData; - } - } - } - - @Override - public byte[] getBinaryValue(byte[] result) { - ensureOpen(); - - if (isBinary) { - if (fieldsData == null) { - // Allocate new buffer if result is null or too small - final byte[] b; - if (result == null || result.length < toRead) - b = new byte[toRead]; - else - b = result; - - IndexInput localFieldsStream = getFieldStream(); - - // Throw this IOException since IndexReader.document does so anyway, so probably not that big of a change for people - // since they are already handling this exception when getting the document - try { - localFieldsStream.seek(pointer); - localFieldsStream.readBytes(b, 0, toRead); - } catch (IOException e) { - throw new FieldReaderException(e); - } - - binaryOffset = 0; - binaryLength = toRead; - if (cacheResult == true){ - fieldsData = b; - } - return b; - } else { - return (byte[]) fieldsData; - } - } else - return null; - } - } - */ } Index: lucene/src/java/org/apache/lucene/index/FieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldsWriter.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/index/FieldsWriter.java (working copy) @@ -140,7 +140,7 @@ int bits = 0; final BytesRef bytes; final String string; - // nocommit -- maybe a field should serialize itself? + // TODO: maybe a field should serialize itself? // this way we don't bake into indexer all these // specific encodings for different fields? and apps // can customize... Index: lucene/src/java/org/apache/lucene/index/IndexableField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexableField.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/index/IndexableField.java (working copy) @@ -21,34 +21,22 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.NumericField; -import org.apache.lucene.document.NumericField.DataType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.values.PerDocFieldValues; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.util.BytesRef; -// nocommit jdocs +// TODO: how to handle versioning here...? -// nocommit maybe abstract class instead...? or maybe we -// have versioned interfaces over time, so indexer can -// consume JARs w/ older doc/field impls? IndexableField1, -// IndexableField2, ...? +/** Represents a single field for indexing. IndexWriter + * consumes Iterable as a document. + * + * @lucene.experimental */ -// nocommit maybe take this further and push analysis into Document - -// nocommit what to do about multi-valued fields? really, -// indexer should not know? - -// nocommit make test case showing how you can index -// Iterable that is not a Document -// instance... - -/** @lucene.experimental */ public interface IndexableField { - // nocommit: attrs? - // nocommit: doc values? - // nocommit: sorted? + // TODO: add attrs to this API? + public String name(); // NOTE: if doc/field impl has the notion of "doc level boost" @@ -57,39 +45,30 @@ public boolean stored(); - // nocommit -- isBinary? public BytesRef binaryValue(BytesRef reuse); public String stringValue(); public Reader readerValue(); - // nocommit -- decouple analyzers here: field impl should - // go and ask analyzer for the token stream, so indexer - // doesn't have to ask for string/reader value and then consult - // analyzer public TokenStream tokenStreamValue(); // Numeric field: public boolean numeric(); - public DataType numericDataType(); + public NumericField.DataType numericDataType(); public Number numericValue(); // If this returns true then we index this field: public boolean indexed(); - // nocommit maybe remove? only needed because stored - // fields records this! (well, and because analysis isn't - // yet decoupled) public boolean tokenized(); public boolean omitNorms(); - public IndexOptions getIndexOptions(); + public IndexOptions indexOptions(); public boolean storeTermVectors(); public boolean storeTermVectorOffsets(); public boolean storeTermVectorPositions(); // doc values - public PerDocFieldValues getDocValues(); - public void setDocValues(PerDocFieldValues docValues); public boolean hasDocValues(); + public PerDocFieldValues docValues(); public ValueType docValuesType(); -} \ No newline at end of file +} Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (working copy) @@ -225,7 +225,7 @@ // easily add it FieldInfo fi = fieldInfos.addOrUpdate(fieldName, field.indexed(), field.storeTermVectors(), field.storeTermVectorPositions(), field.storeTermVectorOffsets(), - field.omitNorms(), false, field.getIndexOptions(), field.docValuesType()); + field.omitNorms(), false, field.indexOptions(), field.docValuesType()); fp = new DocFieldProcessorPerField(this, fi); fp.next = fieldHash[hashPos]; @@ -238,7 +238,7 @@ } else { fieldInfos.addOrUpdate(fp.fieldInfo.name, field.indexed(), field.storeTermVectors(), field.storeTermVectorPositions(), field.storeTermVectorOffsets(), - field.omitNorms(), false, field.getIndexOptions(), field.docValuesType()); + field.omitNorms(), false, field.indexOptions(), field.docValuesType()); } if (thisFieldGen != fp.lastGen) { @@ -264,7 +264,7 @@ } if (field.hasDocValues()) { final DocValuesConsumer docValuesConsumer = docValuesConsumer(docState, fp.fieldInfo); - docValuesConsumer.add(docState.docID, field.getDocValues()); + docValuesConsumer.add(docState.docID, field.docValues()); } } Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -950,11 +950,6 @@ * {@link DocumentStoredFieldVisitor}. */ public abstract void document(int docID, StoredFieldVisitor visitor) throws CorruptIndexException, IOException; - // nocommit -- the new document(int docID) API should - // clearly advertise that only field types/values are - // preserved -- index time metadata like boost, omitNorm, - // IndexOptions, tokenized are not preserved - /** * Returns the stored fields of the nth * Document in this index. This is just @@ -965,11 +960,19 @@ * may yield unspecified results. Usually this is not required, however you * can test if the doc is deleted by checking the {@link * Bits} returned from {@link MultiFields#getLiveDocs}. + * + * NOTE: only the content of a field is returned, + * if that field was stored during indexing. Metadata + * like boost, omitNorm, IndexOptions, tokenized, etc., + * are not preserved. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public org.apache.lucene.document.Document document(int docID) throws CorruptIndexException, IOException { + // TODO: we need a separate StoredField, so that the + // Document returned here contains that class not + // IndexableField + public Document document(int docID) throws CorruptIndexException, IOException { ensureOpen(); final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(); document(docID, visitor); Index: lucene/src/java/org/apache/lucene/document/Field.java =================================================================== --- lucene/src/java/org/apache/lucene/document/Field.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/document/Field.java (working copy) @@ -119,10 +119,6 @@ this.fieldsData = value; } - public boolean isNumeric() { - return false; - } - /** * The value of the field as a String, or null. If null, the Reader value or * binary value is used. Exactly one of stringValue(), readerValue(), and @@ -339,7 +335,7 @@ return type.omitNorms(); } - public IndexOptions getIndexOptions() { + public IndexOptions indexOptions() { return type.indexOptions(); } @@ -355,10 +351,6 @@ return type.storeTermVectorPositions(); } - public boolean lazy() { - return type.lazy(); - } - /** Prints a Field for human consumption. */ @Override public final String toString() { @@ -376,7 +368,7 @@ return result.toString(); } - public PerDocFieldValues getDocValues() { + public PerDocFieldValues docValues() { return docValues; } Index: lucene/src/java/org/apache/lucene/document/NumericField.java =================================================================== --- lucene/src/java/org/apache/lucene/document/NumericField.java (revision 1159961) +++ lucene/src/java/org/apache/lucene/document/NumericField.java (working copy) @@ -309,20 +309,11 @@ return dataType; } - public DataType numericType() { - return dataType; - } - @Override public boolean numeric() { return true; } - @Override - public boolean isNumeric() { - return true; - } - /** * Initializes the field with the supplied long value. * Index: lucene/src/test-framework/org/apache/lucene/index/DocHelper.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (revision 1159961) +++ lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (working copy) @@ -217,9 +217,9 @@ if (f.indexed() && !f.storeTermVectors()) add(notermvector,f); if (f.stored()) add(stored,f); else add(unstored,f); - if (f.getIndexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f); + if (f.indexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f); if (f.omitNorms()) add(noNorms,f); - if (f.getIndexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f); + if (f.indexOptions() == IndexOptions.DOCS_ONLY) add(noTf,f); //if (f.isLazy()) add(lazy, f); } } Index: lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (revision 1159961) +++ lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (working copy) @@ -123,7 +123,7 @@ * @see IndexWriter#addDocument(Iterable) */ public void addDocument(final Iterable doc) throws IOException { - if (doDocValues) { + if (doDocValues && doc instanceof Document) { randomPerDocFieldValues(r, (Document) doc); } if (r.nextInt(5) == 3) { Index: lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (revision 1159961) +++ lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (working copy) @@ -426,7 +426,7 @@ public static void add(Document doc, FieldInfos fieldInfos) { for (IndexableField field : doc) { fieldInfos.addOrUpdate(field.name(), field.indexed(), field.storeTermVectors(), field.storeTermVectorPositions(), - field.storeTermVectorOffsets(), field.omitNorms(), false, field.getIndexOptions(), field.docValuesType()); + field.storeTermVectorOffsets(), field.omitNorms(), false, field.indexOptions(), field.docValuesType()); } }