();
for (Fieldable field : fields) {
if (field.name().equals(name) && (field.isBinary()))
- result.add(field.getBinaryValue());
+ result.add(field.binaryValue(null).bytes);
}
if (result.size() == 0)
@@ -296,7 +303,7 @@
public final byte[] getBinaryValue(String name) {
for (Fieldable field : fields) {
if (field.name().equals(name) && (field.isBinary()))
- return field.getBinaryValue();
+ return field.binaryValue(null).bytes;
}
return null;
}
Index: lucene/src/java/org/apache/lucene/document/Field.java
--- lucene/src/java/org/apache/lucene/document/Field.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/document/Field.java Tue Jun 07 05:56:39 2011 -0400
@@ -20,6 +20,7 @@
import java.io.Reader;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexableDocument; // javadoc
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.util.StringHelper;
@@ -274,7 +275,14 @@
/** The TokesStream for this field to be used when indexing, or null. If null, the Reader value
* or String value is analyzed to produce the indexed tokens. */
public TokenStream tokenStreamValue() { return tokenStream; }
-
+
+ public Number getNumericValue() {
+ return null;
+ }
+
+ public NumericField.DataType getDataType() {
+ return null;
+ }
/** Expert: change the value of this field. This can
* be used during indexing to re-use a single Field
@@ -426,7 +434,7 @@
/**
* Create a tokenized and indexed field that is not stored. Term vectors will
* not be stored. The Reader is read only when the Document is added to the index,
- * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the Reader until {@link IndexWriter#addDocument(IndexableDocument)}
* has been called.
*
* @param name The name of the field
@@ -440,7 +448,7 @@
/**
* Create a tokenized and indexed field that is not stored, optionally with
* storing term vectors. The Reader is read only when the Document is added to the index,
- * i.e. you may not close the Reader until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the Reader until {@link IndexWriter#addDocument(IndexableDocument)}
* has been called.
*
* @param name The name of the field
@@ -471,7 +479,7 @@
* Create a tokenized and indexed field that is not stored. Term vectors will
* not be stored. This is useful for pre-analyzed fields.
* The TokenStream is read only when the Document is added to the index,
- * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(IndexableDocument)}
* has been called.
*
* @param name The name of the field
@@ -486,7 +494,7 @@
* Create a tokenized and indexed field that is not stored, optionally with
* storing term vectors. This is useful for pre-analyzed fields.
* The TokenStream is read only when the Document is added to the index,
- * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(Document)}
+ * i.e. you may not close the TokenStream until {@link IndexWriter#addDocument(IndexableDocument)}
* has been called.
*
* @param name The name of the field
Index: lucene/src/java/org/apache/lucene/document/Fieldable.java
--- lucene/src/java/org/apache/lucene/document/Fieldable.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/document/Fieldable.java Tue Jun 07 05:56:39 2011 -0400
@@ -16,12 +16,11 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.FieldInvertState; // for javadocs
+import org.apache.lucene.index.IndexableField;
import org.apache.lucene.search.PhraseQuery; // for javadocs
import org.apache.lucene.search.spans.SpanQuery; // for javadocs
-
-import java.io.Reader;
+import org.apache.lucene.util.BytesRef; // for javadocs
/**
* Synonymous with {@link Field}.
@@ -33,7 +32,8 @@
*
*
**/
-public interface Fieldable {
+public interface Fieldable extends IndexableField {
+
/** Sets the boost factor hits on this field. This value will be
* multiplied into the score of all hits on this this field of this
* document.
@@ -66,46 +66,46 @@
*
* @see #setBoost(float)
*/
- float getBoost();
+ //float getBoost();
/** Returns the name of the field as an interned string.
* For example "date", "title", "body", ...
*/
- String name();
+ //String name();
/** The value of the field as a String, or null.
*
* For indexing, if isStored()==true, the stringValue() will be used as the stored field value
- * unless isBinary()==true, in which case getBinaryValue() will be used.
+ * unless isBinary()==true, in which case binaryValue() will be used.
*
* If isIndexed()==true and isTokenized()==false, this String value will be indexed as a single token.
* If isIndexed()==true and isTokenized()==true, then tokenStreamValue() will be used to generate indexed tokens if not null,
* else readerValue() will be used to generate indexed tokens if not null, else stringValue() will be used to generate tokens.
*/
- public String stringValue();
+ //public String stringValue();
/** The value of the field as a Reader, which can be used at index time to generate indexed tokens.
* @see #stringValue()
*/
- public Reader readerValue();
+ //public Reader readerValue();
/** The TokenStream for this field to be used when indexing, or null.
* @see #stringValue()
*/
- public TokenStream tokenStreamValue();
+ //public TokenStream tokenStreamValue();
/** True if the value of the field is to be stored in the index for return
with search hits. */
- boolean isStored();
+ //boolean isStored();
/** True if the value of the field is to be indexed, so that it may be
searched on. */
- boolean isIndexed();
+ //boolean isIndexed();
/** True if the value of the field should be tokenized as text prior to
indexing. Un-tokenized fields are indexed as a single word and may not be
Reader-valued. */
- boolean isTokenized();
+ //boolean isTokenized();
/** True if the term or terms used to index this field are stored as a term
* vector, available from {@link org.apache.lucene.index.IndexReader#getTermFreqVector(int,String)}.
@@ -115,24 +115,24 @@
*
* @see org.apache.lucene.index.IndexReader#getTermFreqVector(int, String)
*/
- boolean isTermVectorStored();
+ //boolean isTermVectorStored();
/**
* True if terms are stored as term vector together with their offsets
* (start and end positon in source text).
*/
- boolean isStoreOffsetWithTermVector();
+ //boolean isStoreOffsetWithTermVector();
/**
* True if terms are stored as term vector together with their token positions.
*/
- boolean isStorePositionWithTermVector();
+ //boolean isStorePositionWithTermVector();
/** True if the value of the field is stored as binary */
- boolean isBinary();
+ boolean isBinary();
/** True if norms are omitted for this indexed field */
- boolean getOmitNorms();
+ //boolean getOmitNorms();
/** Expert:
*
@@ -143,7 +143,7 @@
/**
* Indicates whether a Field is Lazy or not. The semantics of Lazy loading are such that if a Field is lazily loaded, retrieving
- * it's values via {@link #stringValue()} or {@link #getBinaryValue()} is only valid as long as the {@link org.apache.lucene.index.IndexReader} that
+ * it's values via {@link #stringValue()} or {@link #binaryValue(BytesRef)} is only valid as long as the {@link org.apache.lucene.index.IndexReader} that
* retrieved the {@link Document} is still open.
*
* @return true if this field can be loaded lazily
@@ -155,14 +155,14 @@
* returned value is undefined
* @return index of the first character in byte[] segment that represents this Field value
*/
- abstract int getBinaryOffset();
+ //abstract int getBinaryOffset();
/**
* Returns length of byte[] segment that is used as value, if Field is not binary
* returned value is undefined
* @return length of byte[] segment that represents this Field value
*/
- abstract int getBinaryLength();
+ //abstract int getBinaryLength();
/**
* Return the raw byte[] for the binary field. Note that
@@ -171,17 +171,24 @@
* returned array belong to the field.
* @return reference to the Field value as byte[].
*/
- abstract byte[] getBinaryValue();
+ //abstract byte[] getBinaryValue();
+
+ // nocommit api break
+ //abstract BytesRef binaryValue();
+
+ //abstract NumericField.DataType getDataType();
+
+ //abstract Number getNumericValue();
/**
* Return the raw byte[] for the binary field. Note that
- * you must also call {@link #getBinaryLength} and {@link
- * #getBinaryOffset} to know which range of bytes in this
+ * you must also call {@link #binaryValue}
+ * to know which range of bytes in this
* returned array belong to the field.
* About reuse: if you pass in the result byte[] and it is
* used, likely the underlying implementation will hold
* onto this byte[] and return it in future calls to
- * {@link #getBinaryValue()}.
+ * {@link #binaryValue(BytesRef)}.
* So if you subsequently re-use the same byte[] elsewhere
* it will alter this Fieldable's value.
* @param result User defined buffer that will be used if
@@ -189,10 +196,11 @@
* buffer is allocated
* @return reference to the Field value as byte[].
*/
- abstract byte[] getBinaryValue(byte[] result);
+ // nocommit -- remove this too; add resuse param to binaryValue
+ //abstract byte[] getBinaryValue(byte[] result);
/** @see #setOmitTermFreqAndPositions */
- boolean getOmitTermFreqAndPositions();
+ //boolean getOmitTermFreqAndPositions();
/** Expert:
*
Index: lucene/src/java/org/apache/lucene/document/NumericField.java
--- lucene/src/java/org/apache/lucene/document/NumericField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/document/NumericField.java Tue Jun 07 05:56:39 2011 -0400
@@ -226,12 +226,6 @@
}
/** Returns always null for numeric fields */
- @Override
- public byte[] getBinaryValue(byte[] result){
- return null;
- }
-
- /** Returns always null for numeric fields */
public Reader readerValue() {
return null;
}
@@ -260,6 +254,10 @@
public DataType getDataType() {
return type;
}
+
+ public boolean isNumeric() {
+ return true;
+ }
/**
* Initializes the field with the supplied long value.
Index: lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
--- lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -18,11 +18,10 @@
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
abstract class DocFieldConsumerPerField {
/** Processes all occurrences of a single field */
- abstract void processFields(Fieldable[] fields, int count) throws IOException;
+ abstract void processFields(IndexableField[] fields, int count) throws IOException;
abstract void abort();
abstract FieldInfo getFieldInfo();
}
Index: lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java
--- lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocFieldConsumersPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -18,7 +18,6 @@
*/
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
final class DocFieldConsumersPerField extends DocFieldConsumerPerField {
@@ -35,7 +34,7 @@
}
@Override
- public void processFields(Fieldable[] fields, int count) throws IOException {
+ public void processFields(IndexableField[] fields, int count) throws IOException {
one.processFields(fields, count);
two.processFields(fields, count);
}
Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
--- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Jun 07 05:56:39 2011 -0400
@@ -22,11 +22,8 @@
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
@@ -181,22 +178,16 @@
consumer.startDocument();
fieldsWriter.startDocument();
- final Document doc = docState.doc;
-
fieldCount = 0;
final int thisFieldGen = fieldGen++;
- final List docFields = doc.getFields();
- final int numDocFields = docFields.size();
-
// Absorb any new fields first seen in this document.
// Also absorb any changes to fields we had already
// seen before (eg suddenly turning on norms or
// vectors, etc.):
- for(int i=0;i= fieldHash.length/2)
+ if (totalFieldCount >= fieldHash.length/2) {
rehash();
+ }
} else {
fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
+ field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
}
if (thisFieldGen != fp.lastGen) {
Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
--- lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -34,17 +33,17 @@
int lastGen = -1;
int fieldCount;
- Fieldable[] fields = new Fieldable[1];
+ IndexableField[] fields = new IndexableField[1];
public DocFieldProcessorPerField(final DocFieldProcessor docFieldProcessor, final FieldInfo fieldInfo) {
this.consumer = docFieldProcessor.consumer.addField(fieldInfo);
this.fieldInfo = fieldInfo;
}
- public void addField(Fieldable field) {
+ public void addField(IndexableField field) {
if (fieldCount == fields.length) {
int newSize = ArrayUtil.oversize(fieldCount + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(fields, 0, newArray, 0, fieldCount);
fields = newArray;
}
Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
--- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -19,7 +19,6 @@
import java.io.IOException;
import java.io.Reader;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
@@ -61,7 +60,7 @@
}
@Override
- public void processFields(final Fieldable[] fields,
+ public void processFields(final IndexableField[] fields,
final int count) throws IOException {
fieldState.reset(docState.doc.getBoost());
@@ -70,7 +69,7 @@
for(int i=0;i 0)
fieldState.position += docState.analyzer.getPositionIncrementGap(fieldInfo.name);
+ // nocommit -- this logic should be outside of
+ // indexer
+
if (!field.isTokenized()) { // un-tokenized field
String stringValue = field.stringValue();
final int valueLength = stringValue.length();
@@ -103,17 +105,17 @@
final TokenStream stream;
final TokenStream streamValue = field.tokenStreamValue();
- if (streamValue != null)
+ if (streamValue != null) {
stream = streamValue;
- else {
+ } else {
// the field does not have a TokenStream,
// so we have to obtain one from the analyzer
final Reader reader; // find or make Reader
final Reader readerValue = field.readerValue();
- if (readerValue != null)
+ if (readerValue != null) {
reader = readerValue;
- else {
+ } else {
String stringValue = field.stringValue();
if (stringValue == null) {
throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
--- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -27,7 +27,6 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
@@ -321,7 +320,7 @@
return maybeMerge;
}
- boolean updateDocuments(final Iterable docs, final Analyzer analyzer,
+ boolean updateDocuments(final Iterable docs, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
@@ -352,7 +351,7 @@
return postUpdate(flushingDWPT, maybeMerge);
}
- boolean updateDocument(final Document doc, final Analyzer analyzer,
+ boolean updateDocument(final IndexableDocument doc, final Analyzer analyzer,
final Term delTerm) throws CorruptIndexException, IOException {
boolean maybeMerge = preUpdate();
Index: lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
--- lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Jun 07 05:56:39 2011 -0400
@@ -26,7 +26,6 @@
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.store.Directory;
@@ -87,7 +86,7 @@
PrintStream infoStream;
SimilarityProvider similarityProvider;
int docID;
- Document doc;
+ IndexableDocument doc;
String maxTermPrefix;
DocState(DocumentsWriterPerThread docWriter) {
@@ -209,7 +208,7 @@
return retval;
}
- public void updateDocument(Document doc, Analyzer analyzer, Term delTerm) throws IOException {
+ public void updateDocument(IndexableDocument doc, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocument start");
assert deleteQueue != null;
docState.doc = doc;
@@ -253,7 +252,7 @@
finishDocument(delTerm);
}
- public int updateDocuments(Iterable docs, Analyzer analyzer, Term delTerm) throws IOException {
+ public int updateDocuments(Iterable docs, Analyzer analyzer, Term delTerm) throws IOException {
assert writer.testPoint("DocumentsWriterPerThread addDocuments start");
assert deleteQueue != null;
docState.analyzer = analyzer;
@@ -265,7 +264,7 @@
int docCount = 0;
try {
- for(Document doc : docs) {
+ for(IndexableDocument doc : docs) {
docState.doc = doc;
docState.docID = numDocsInRAM;
docCount++;
Index: lucene/src/java/org/apache/lucene/index/FieldsReader.java
--- lucene/src/java/org/apache/lucene/index/FieldsReader.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/FieldsReader.java Tue Jun 07 05:56:39 2011 -0400
@@ -17,23 +17,23 @@
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Reader;
+
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.AbstractField;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CloseableThreadLocal;
-import java.io.IOException;
-import java.io.Reader;
-
/**
* Class responsible for access to stored document fields.
*
@@ -407,10 +407,10 @@
}
/**
- * A Lazy implementation of Fieldable that defers loading of fields until asked for, instead of when the Document is
+ * A Lazy field implementation that defers loading of fields until asked for, instead of when the Document is
* loaded.
*/
- private class LazyField extends AbstractField implements Fieldable {
+ private class LazyField extends AbstractField {
private int toRead;
private long pointer;
private final boolean cacheResult;
@@ -437,6 +437,14 @@
lazy = true;
}
+ public Number getNumericValue() {
+ return null;
+ }
+
+ public NumericField.DataType getDataType() {
+ return null;
+ }
+
private IndexInput getFieldStream() {
IndexInput localFieldsStream = fieldsStreamTL.get();
if (localFieldsStream == null) {
@@ -491,8 +499,7 @@
}
}
- @Override
- public byte[] getBinaryValue(byte[] result) {
+ private byte[] getBinaryValue(byte[] result) {
ensureOpen();
if (isBinary) {
@@ -527,5 +534,15 @@
} else
return null;
}
+
+ @Override
+ public BytesRef binaryValue(BytesRef reuse) {
+ final byte[] bytes = getBinaryValue(reuse != null ? reuse.bytes : null);
+ if (bytes != null) {
+ return new BytesRef(bytes, 0, bytes.length);
+ } else {
+ return null;
+ }
+ }
}
}
Index: lucene/src/java/org/apache/lucene/index/FieldsWriter.java
--- lucene/src/java/org/apache/lucene/index/FieldsWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/FieldsWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -17,14 +17,11 @@
*/
import java.io.IOException;
-import java.util.List;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.NumericField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
final class FieldsWriter {
@@ -137,15 +134,16 @@
}
}
- final void writeField(int fieldNumber, Fieldable field) throws IOException {
+ final void writeField(int fieldNumber, IndexableField field) throws IOException {
fieldsStream.writeVInt(fieldNumber);
int bits = 0;
- if (field.isTokenized())
+ if (field.isIndexed() && field.isTokenized()) {
bits |= FIELD_IS_TOKENIZED;
- if (field.isBinary())
- bits |= FIELD_IS_BINARY;
- if (field instanceof NumericField) {
- switch (((NumericField) field).getDataType()) {
+ }
+ final BytesRef bytes;
+ final String string;
+ if (field.isNumeric()) {
+ switch (field.getDataType()) {
case INT:
bits |= FIELD_IS_NUMERIC_INT; break;
case LONG:
@@ -157,23 +155,31 @@
default:
assert false : "Should never get here";
}
+ string = null;
+ bytes = null;
+ } else {
+ bytes = field.binaryValue(null);
+ if (bytes != null) {
+ bits |= FIELD_IS_BINARY;
+ string = null;
+ } else {
+ string = field.stringValue();
+ }
}
+
fieldsStream.writeByte((byte) bits);
- if (field.isBinary()) {
- final byte[] data;
- final int len;
- final int offset;
- data = field.getBinaryValue();
- len = field.getBinaryLength();
- offset = field.getBinaryOffset();
-
- fieldsStream.writeVInt(len);
- fieldsStream.writeBytes(data, offset, len);
- } else if (field instanceof NumericField) {
- final NumericField nf = (NumericField) field;
- final Number n = nf.getNumericValue();
- switch (nf.getDataType()) {
+ if (bytes != null) {
+ fieldsStream.writeVInt(bytes.length);
+ fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ } else if (string != null) {
+ fieldsStream.writeString(field.stringValue());
+ } else {
+ final Number n = field.getNumericValue();
+ if (n == null) {
+ throw new IllegalArgumentException("field " + field.name() + " is stored but does not have binaryValue, stringValue nor numericValue");
+ }
+ switch (field.getDataType()) {
case INT:
fieldsStream.writeInt(n.intValue()); break;
case LONG:
@@ -185,8 +191,6 @@
default:
assert false : "Should never get here";
}
- } else {
- fieldsStream.writeString(field.stringValue());
}
}
@@ -206,21 +210,21 @@
assert fieldsStream.getFilePointer() == position;
}
- final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException {
+ final void addDocument(IndexableDocument doc, FieldInfos fieldInfos) throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
int storedCount = 0;
- List fields = doc.getFields();
- for (Fieldable field : fields) {
- if (field.isStored())
- storedCount++;
+ for (IndexableField field : doc) {
+ if (field.isStored()) {
+ storedCount++;
+ }
}
fieldsStream.writeVInt(storedCount);
-
- for (Fieldable field : fields) {
- if (field.isStored())
+ for (IndexableField field : doc) {
+ if (field.isStored()) {
writeField(fieldInfos.fieldNumber(field.name()), field);
+ }
}
}
}
Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
--- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -22,7 +22,6 @@
import java.util.Map;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.PostingsConsumer;
import org.apache.lucene.index.codecs.TermStats;
@@ -81,15 +80,17 @@
}
@Override
- boolean start(Fieldable[] fields, int count) {
- for(int i=0;i
- In either case, documents are added with {@link #addDocument(Document)
+
In either case, documents are added with {@link #addDocument(IndexableDocument)
addDocument} and removed with {@link #deleteDocuments(Term)} or {@link
#deleteDocuments(Query)}. A document can be updated with {@link
- #updateDocument(Term, Document) updateDocument} (which just deletes
+ #updateDocument(Term, IndexableDocument) updateDocument} (which just deletes
and then adds the entire document). When finished adding, deleting
and updating documents, {@link #close() close} should be called.
@@ -1210,7 +1209,7 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc) throws CorruptIndexException, IOException {
+ public void addDocument(IndexableDocument doc) throws CorruptIndexException, IOException {
addDocument(doc, analyzer);
}
@@ -1218,7 +1217,7 @@
* Adds a document to this index, using the provided analyzer instead of the
* value of {@link #getAnalyzer()}.
*
- * See {@link #addDocument(Document)} for details on
+ *
See {@link #addDocument(IndexableDocument)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.
*
@@ -1229,7 +1228,7 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocument(IndexableDocument doc, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocument(null, doc, analyzer);
}
@@ -1247,7 +1246,7 @@
* compression), in which case you may need to fully
* re-index your documents at that time.
*
- * See {@link #addDocument(Document)} for details on
+ *
See {@link #addDocument(IndexableDocument)} for details on
* index and IndexWriter state after an Exception, and
* flushing/merging temporary free space requirements.
*
@@ -1267,7 +1266,7 @@
*
* @lucene.experimental
*/
- public void addDocuments(Iterable docs) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable> docs) throws CorruptIndexException, IOException {
addDocuments(docs, analyzer);
}
@@ -1282,7 +1281,7 @@
*
* @lucene.experimental
*/
- public void addDocuments(Iterable docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void addDocuments(Iterable> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
updateDocuments(null, docs, analyzer);
}
@@ -1299,7 +1298,7 @@
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable docs) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable> docs) throws CorruptIndexException, IOException {
updateDocuments(delTerm, docs, analyzer);
}
@@ -1317,7 +1316,7 @@
*
* @lucene.experimental
*/
- public void updateDocuments(Term delTerm, Iterable docs, Analyzer analyzer) throws CorruptIndexException, IOException {
+ public void updateDocuments(Term delTerm, Iterable> docs, Analyzer analyzer) throws CorruptIndexException, IOException {
ensureOpen();
try {
boolean success = false;
@@ -1440,7 +1439,7 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc) throws CorruptIndexException, IOException {
+ public void updateDocument(Term term, IndexableDocument doc) throws CorruptIndexException, IOException {
ensureOpen();
updateDocument(term, doc, getAnalyzer());
}
@@ -1463,7 +1462,7 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public void updateDocument(Term term, Document doc, Analyzer analyzer)
+ public void updateDocument(Term term, IndexableDocument doc, Analyzer analyzer)
throws CorruptIndexException, IOException {
ensureOpen();
try {
@@ -2852,7 +2851,7 @@
DocumentsWriter getDocsWriter() {
boolean test = false;
assert test = true;
- return test?docWriter: null;
+ return test ? docWriter : null;
}
/** Expert: Return the number of documents currently
Index: lucene/src/java/org/apache/lucene/index/IndexableDocument.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ lucene/src/java/org/apache/lucene/index/IndexableDocument.java Tue Jun 07 05:56:39 2011 -0400
@@ -0,0 +1,27 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Iterator;
+
+public interface IndexableDocument extends Iterable {
+
+ float getBoost();
+
+ Iterator iterator();
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/index/IndexableField.java
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ lucene/src/java/org/apache/lucene/index/IndexableField.java Tue Jun 07 05:56:39 2011 -0400
@@ -0,0 +1,68 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.document.NumericField;
+import org.apache.lucene.util.BytesRef;
+
+// nocommit jdocs
+
+// nocommit maybe take this further and push analysis into Document
+
+// nocommit what to do about multi-valued fields? really,
+// indexer should not know?
+
+/** @lucene.experimental */
+public interface IndexableField {
+ // nocommit: attrs?
+ // nocommit: doc values?
+ // nocommit: sorted?
+
+ public String name();
+
+ // nocommit -- make sure doc multplies its own boost in here:
+ public float getBoost();
+
+ public boolean isStored();
+
+ // nocommit -- isBinary?
+ public BytesRef binaryValue(BytesRef reuse);
+ public String stringValue();
+ public Reader readerValue();
+ public TokenStream tokenStreamValue();
+
+ // Numeric field:
+ public boolean isNumeric();
+ public NumericField.DataType getDataType();
+ public Number getNumericValue();
+
+ // If this returns non-null then we index this field:
+ public boolean isIndexed();
+ // nocommit maybe remove? only needed because stored
+ // fields records this!
+ public boolean isTokenized();
+ public boolean getOmitNorms();
+ public boolean getOmitTermFreqAndPositions();
+
+ public boolean isTermVectorStored();
+ public boolean isStoreOffsetWithTermVector();
+ public boolean isStorePositionWithTermVector();
+}
\ No newline at end of file
Index: lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
--- lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -19,24 +19,22 @@
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
-
abstract class InvertedDocConsumerPerField {
- // Called once per field, and is given all Fieldable
+ // Called once per field, and is given all IndexableField
// occurrences for this field in the document. Return
// true if you wish to see inverted tokens for these
// fields:
- abstract boolean start(Fieldable[] fields, int count) throws IOException;
+ abstract boolean start(IndexableField[] fields, int count) throws IOException;
// Called before a field instance is being processed
- abstract void start(Fieldable field);
+ abstract void start(IndexableField field);
// Called once per inverted token
abstract void add() throws IOException;
- // Called once per field per document, after all Fieldable
- // occurrences are inverted
+ // Called once per field per document, after all IndexableFields
+ // are inverted
abstract void finish() throws IOException;
// Called on hitting an aborting exception
Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java
--- lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -19,7 +19,6 @@
import java.io.IOException;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
@@ -40,12 +39,12 @@
}
private int numStoredFields;
- private Fieldable[] storedFields;
+ private IndexableField[] storedFields;
private int[] fieldNumbers;
public void reset() {
numStoredFields = 0;
- storedFields = new Fieldable[1];
+ storedFields = new IndexableField[1];
fieldNumbers = new int[1];
}
@@ -122,10 +121,10 @@
assert docWriter.writer.testPoint("StoredFieldsWriter.finishDocument end");
}
- public void addField(Fieldable field, FieldInfo fieldInfo) throws IOException {
+ public void addField(IndexableField field, FieldInfo fieldInfo) throws IOException {
if (numStoredFields == storedFields.length) {
int newSize = ArrayUtil.oversize(numStoredFields + 1, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
- Fieldable[] newArray = new Fieldable[newSize];
+ IndexableField[] newArray = new IndexableField[newSize];
System.arraycopy(storedFields, 0, newArray, 0, numStoredFields);
storedFields = newArray;
}
Index: lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
--- lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java Tue Jun 07 05:56:39 2011 -0400
@@ -20,7 +20,6 @@
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -55,13 +54,13 @@
}
@Override
- boolean start(Fieldable[] fields, int count) {
+ boolean start(IndexableField[] fields, int count) {
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
for(int i=0;istate.getLength() is small.
*
* Note that the return values are computed under
- * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)}
+ * {@link org.apache.lucene.index.IndexWriter#addDocument(IndexableDocument)}
* and then stored using
* {@link #encodeNormValue(float)}.
* Thus they have limited precision, and documents
Index: lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
--- lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -24,7 +24,6 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexWriter; // javadoc
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -96,19 +95,19 @@
/**
* Adds a Document.
- * @see IndexWriter#addDocument(Document)
+ * @see IndexWriter#addDocument(IndexableDocument)
*/
- public void addDocument(final Document doc) throws IOException {
+ public void addDocument(final IndexableDocument doc) throws IOException {
if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing.
- w.addDocuments(new Iterable() {
+ w.addDocuments(new Iterable>() {
// @Override -- not until Java 1.6
- public Iterator iterator() {
- return new Iterator() {
+ public Iterator> iterator() {
+ return new Iterator>() {
boolean done;
// @Override -- not until Java 1.6
@@ -122,7 +121,7 @@
}
// @Override -- not until Java 1.6
- public Document next() {
+ public IndexableDocument next() {
if (done) {
throw new IllegalStateException();
}
@@ -152,27 +151,27 @@
}
}
- public void addDocuments(Iterable docs) throws IOException {
+ public void addDocuments(Iterable> docs) throws IOException {
w.addDocuments(docs);
maybeCommit();
}
- public void updateDocuments(Term delTerm, Iterable docs) throws IOException {
+ public void updateDocuments(Term delTerm, Iterable> docs) throws IOException {
w.updateDocuments(delTerm, docs);
maybeCommit();
}
/**
* Updates a document.
- * @see IndexWriter#updateDocument(Term, Document)
+ * @see IndexWriter#updateDocument(Term, IndexableDocument)
*/
- public void updateDocument(Term t, final Document doc) throws IOException {
+ public void updateDocument(Term t, final IndexableDocument doc) throws IOException {
if (r.nextInt(5) == 3) {
- w.updateDocuments(t, new Iterable() {
+ w.updateDocuments(t, new Iterable>() {
// @Override -- not until Java 1.6
- public Iterator iterator() {
- return new Iterator() {
+ public Iterator> iterator() {
+ return new Iterator>() {
boolean done;
// @Override -- not until Java 1.6
@@ -186,7 +185,7 @@
}
// @Override -- not until Java 1.6
- public Document next() {
+ public IndexableDocument next() {
if (done) {
throw new IllegalStateException();
}
Index: lucene/src/test/org/apache/lucene/index/TestFieldsReader.java
--- lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/test/org/apache/lucene/index/TestFieldsReader.java Tue Jun 07 05:56:39 2011 -0400
@@ -148,10 +148,10 @@
assertTrue("field is null and it shouldn't be", field != null);
assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
- byte [] bytes = field.getBinaryValue();
+ byte [] bytes = field.binaryValue(null).bytes;
assertTrue("bytes is null and it shouldn't be", bytes != null);
assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
- assertTrue("calling binaryValue() twice should give same reference", field.getBinaryValue() == field.getBinaryValue());
+ assertTrue("calling binaryValue() twice should give same reference", field.binaryValue(null).bytes == field.binaryValue(null).bytes);
for (int i = 0; i < bytes.length; i++) {
assertTrue("byte[" + i + "] is mismatched", bytes[i] == DocHelper.LAZY_FIELD_BINARY_BYTES[i]);
@@ -217,9 +217,9 @@
field = doc.getFieldable(DocHelper.LAZY_FIELD_BINARY_KEY);
assertTrue("field is null and it shouldn't be", field != null);
assertTrue("stringValue isn't null for lazy binary field", field.stringValue() == null);
- assertTrue("calling binaryValue() twice should give different references", field.getBinaryValue() != field.getBinaryValue());
+ assertTrue("calling binaryValue() twice should give different references", field.binaryValue(null).bytes != field.binaryValue(null).bytes);
- byte [] bytes = field.getBinaryValue();
+ byte [] bytes = field.binaryValue(null).bytes;
assertTrue("bytes is null and it shouldn't be", bytes != null);
assertTrue("", DocHelper.LAZY_FIELD_BINARY_BYTES.length == bytes.length);
for (int i = 0; i < bytes.length; i++) {
@@ -375,9 +375,9 @@
assertTrue(f1.isBinary());
assertTrue(!f3.isBinary());
assertTrue(fb.isBinary());
- assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.getBinaryValue());
+ assertSizeEquals(2*DocHelper.FIELD_1_TEXT.length(), f1.binaryValue(null).bytes);
assertEquals(DocHelper.FIELD_3_TEXT, f3.stringValue());
- assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.getBinaryValue());
+ assertSizeEquals(DocHelper.LAZY_FIELD_BINARY_BYTES.length, fb.binaryValue(null).bytes);
reader.close();
}
Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java
--- lucene/src/test/org/apache/lucene/index/TestIndexReader.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java Tue Jun 07 05:56:39 2011 -0400
@@ -349,10 +349,10 @@
assertEquals(1, fields.length);
Field b1 = fields[0];
assertTrue(b1.isBinary());
- byte[] data1 = b1.getBinaryValue();
- assertEquals(bin.length, b1.getBinaryLength());
+ BytesRef bytesRef = b1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
Set lazyFields = new HashSet();
lazyFields.add("bin1");
@@ -363,11 +363,11 @@
assertEquals(1, fieldables.length);
Fieldable fb1 = fieldables[0];
assertTrue(fb1.isBinary());
- assertEquals(bin.length, fb1.getBinaryLength());
- data1 = fb1.getBinaryValue();
- assertEquals(bin.length, fb1.getBinaryLength());
+ bytesRef = fb1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.bytes.length);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + fb1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
reader.close();
// force optimize
@@ -383,10 +383,10 @@
assertEquals(1, fields.length);
b1 = fields[0];
assertTrue(b1.isBinary());
- data1 = b1.getBinaryValue();
- assertEquals(bin.length, b1.getBinaryLength());
+ bytesRef = b1.binaryValue(null);
+ assertEquals(bin.length, bytesRef.length);
for (int i = 0; i < bin.length; i++) {
- assertEquals(bin[i], data1[i + b1.getBinaryOffset()]);
+ assertEquals(bin[i], bytesRef.bytes[i + bytesRef.offset]);
}
reader.close();
dir.close();
Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
--- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -971,11 +971,11 @@
Document doc = new Document();
Field f = new Field("binary", b, 10, 17);
- byte[] bx = f.getBinaryValue();
+ byte[] bx = f.binaryValue(null).bytes;
assertTrue(bx != null);
assertEquals(50, bx.length);
- assertEquals(10, f.getBinaryOffset());
- assertEquals(17, f.getBinaryLength());
+ assertEquals(10, f.binaryValue(null).offset);
+ assertEquals(17, f.binaryValue(null).length);
doc.add(f);
w.addDocument(doc);
w.close();
@@ -983,7 +983,7 @@
IndexReader ir = IndexReader.open(dir, true);
doc = ir.document(0);
f = doc.getField("binary");
- b = f.getBinaryValue();
+ b = f.binaryValue(null).bytes;
assertTrue(b != null);
assertEquals(17, b.length, 17);
assertEquals(87, b[0]);
@@ -1238,7 +1238,7 @@
IndexReader ir = IndexReader.open(dir, true);
doc = ir.document(0);
f = doc.getField("binary");
- b = f.getBinaryValue();
+ b = f.binaryValue(null).bytes;
assertTrue(b != null);
assertEquals(17, b.length, 17);
assertEquals(87, b[0]);
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java
--- modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java Mon Jun 06 14:39:26 2011 -0400
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/LimitTokenCountAnalyzer.java Tue Jun 07 05:56:39 2011 -0400
@@ -17,9 +17,9 @@
* limitations under the License.
*/
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexableField;
import java.io.Reader;
import java.io.IOException;
@@ -60,7 +60,7 @@
}
@Override
- public int getOffsetGap(Fieldable field) {
+ public int getOffsetGap(IndexableField field) {
return delegate.getOffsetGap(field);
}
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java
--- modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java Mon Jun 06 14:39:26 2011 -0400
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java Tue Jun 07 05:56:39 2011 -0400
@@ -19,7 +19,7 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.IndexableField;
import java.io.Reader;
import java.io.IOException;
@@ -119,10 +119,11 @@
/** Return the offsetGap from the analyzer assigned to field */
@Override
- public int getOffsetGap(Fieldable field) {
+ public int getOffsetGap(IndexableField field) {
Analyzer analyzer = analyzerMap.get(field.name());
- if (analyzer == null)
+ if (analyzer == null) {
analyzer = defaultAnalyzer;
+ }
return analyzer.getOffsetGap(field);
}
Index: solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
--- solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Mon Jun 06 14:39:26 2011 -0400
+++ solr/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Tue Jun 07 05:56:39 2011 -0400
@@ -253,9 +253,9 @@
// TODO: this really should be "stored"
f.add( "internal", fieldable.stringValue() ); // may be a binary number
- byte[] arr = fieldable.getBinaryValue();
- if (arr != null) {
- f.add( "binary", Base64.byteArrayToBase64(arr, 0, arr.length));
+ BytesRef bytes = fieldable.binaryValue(null);
+ if (bytes != null) {
+ f.add( "binary", Base64.byteArrayToBase64(bytes.bytes, bytes.offset, bytes.length));
}
f.add( "boost", fieldable.getBoost() );
f.add( "docFreq", t.text()==null ? 0 : reader.docFreq( t ) ); // this can be 0 for non-indexed fields
Index: solr/src/java/org/apache/solr/response/BinaryResponseWriter.java
--- solr/src/java/org/apache/solr/response/BinaryResponseWriter.java Mon Jun 06 14:39:26 2011 -0400
+++ solr/src/java/org/apache/solr/response/BinaryResponseWriter.java Tue Jun 07 05:56:39 2011 -0400
@@ -165,7 +165,7 @@
if(sf != null) ft =sf.getType();
Object val;
if (ft == null) { // handle fields not in the schema
- if (f.isBinary()) val = f.getBinaryValue();
+ if (f.isBinary()) val = f.binaryValue(null).bytes;
else val = f.stringValue();
} else {
try {
Index: solr/src/java/org/apache/solr/schema/BinaryField.java
--- solr/src/java/org/apache/solr/schema/BinaryField.java Mon Jun 06 14:39:26 2011 -0400
+++ solr/src/java/org/apache/solr/schema/BinaryField.java Tue Jun 07 05:56:39 2011 -0400
@@ -17,15 +17,16 @@
package org.apache.solr.schema;
-import org.apache.solr.response.TextResponseWriter;
-import org.apache.solr.common.util.Base64;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.search.SortField;
-
import java.io.IOException;
import java.nio.ByteBuffer;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.search.SortField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.util.Base64;
+import org.apache.solr.response.TextResponseWriter;
+
public class BinaryField extends FieldType {
@@ -51,7 +52,8 @@
@Override
public ByteBuffer toObject(Fieldable f) {
- return ByteBuffer.wrap(f.getBinaryValue(), f.getBinaryOffset(), f.getBinaryLength() ) ;
+ BytesRef bytes = f.binaryValue(null);
+ return ByteBuffer.wrap(bytes.bytes, bytes.offset, bytes.length);
}
@Override
Index: solr/src/java/org/apache/solr/schema/TrieField.java
--- solr/src/java/org/apache/solr/schema/TrieField.java Mon Jun 06 14:39:26 2011 -0400
+++ solr/src/java/org/apache/solr/schema/TrieField.java Tue Jun 07 05:56:39 2011 -0400
@@ -109,19 +109,19 @@
return (type == TrieTypes.DATE) ? new Date(val.longValue()) : val;
} else {
// the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding:
- final byte[] arr = f.getBinaryValue();
- if (arr==null) return badFieldString(f);
+ final BytesRef bytes = f.binaryValue(null);
+ if (bytes==null) return badFieldString(f);
switch (type) {
case INTEGER:
- return toInt(arr);
+ return toInt(bytes.bytes);
case FLOAT:
- return Float.intBitsToFloat(toInt(arr));
+ return Float.intBitsToFloat(toInt(bytes.bytes));
case LONG:
- return toLong(arr);
+ return toLong(bytes.bytes);
case DOUBLE:
- return Double.longBitsToDouble(toLong(arr));
+ return Double.longBitsToDouble(toLong(bytes.bytes));
case DATE:
- return new Date(toLong(arr));
+ return new Date(toLong(bytes.bytes));
default:
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + f.name());
}
@@ -432,31 +432,31 @@
}
} else {
// the following code is "deprecated" and only to support pre-3.2 indexes using the old BinaryField encoding:
- final byte[] arr = f.getBinaryValue();
- if (arr==null)
+ final BytesRef bytesRef = f.binaryValue(null);
+ if (bytesRef==null)
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Invalid field contents: "+f.name());
switch (type) {
case INTEGER:
- NumericUtils.intToPrefixCoded(toInt(arr), 0, bytes);
+ NumericUtils.intToPrefixCoded(toInt(bytesRef.bytes), 0, bytes);
break;
case FLOAT: {
// WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
// copied from NumericUtils to not convert to/from float two times
// code in next 2 lines is identical to: int v = NumericUtils.floatToSortableInt(Float.intBitsToFloat(toInt(arr)));
- int v = toInt(arr);
+ int v = toInt(bytesRef.bytes);
if (v<0) v ^= 0x7fffffff;
NumericUtils.intToPrefixCoded(v, 0, bytes);
break;
}
case LONG: //fallthrough!
case DATE:
- NumericUtils.longToPrefixCoded(toLong(arr), 0, bytes);
+ NumericUtils.longToPrefixCoded(toLong(bytesRef.bytes), 0, bytes);
break;
case DOUBLE: {
// WARNING: Code Duplication! Keep in sync with o.a.l.util.NumericUtils!
// copied from NumericUtils to not convert to/from double two times
// code in next 2 lines is identical to: long v = NumericUtils.doubleToSortableLong(Double.longBitsToDouble(toLong(arr)));
- long v = toLong(arr);
+ long v = toLong(bytesRef.bytes);
if (v<0) v ^= 0x7fffffffffffffffL;
NumericUtils.longToPrefixCoded(v, 0, bytes);
break;
Index: solr/src/test/org/apache/solr/schema/PolyFieldTest.java
--- solr/src/test/org/apache/solr/schema/PolyFieldTest.java Mon Jun 06 14:39:26 2011 -0400
+++ solr/src/test/org/apache/solr/schema/PolyFieldTest.java Tue Jun 07 05:56:39 2011 -0400
@@ -88,7 +88,7 @@
//first two fields contain the values, third is just stored and contains the original
for (int i = 0; i < 3; i++) {
boolean hasValue = fields[1].tokenStreamValue() != null
- || fields[1].getBinaryValue() != null
+ || fields[1].binaryValue(null) != null
|| fields[1].stringValue() != null;
assertTrue("Doesn't have a value: " + fields[1], hasValue);
}