diff -crX exclude.files LuceneTrunkFreshPull/src/java/org/apache/lucene/document/Field.java LuceneTrunk/src/java/org/apache/lucene/document/Field.java
*** LuceneTrunkFreshPull/src/java/org/apache/lucene/document/Field.java 2005-12-07 21:30:01.000000000 -1000
--- LuceneTrunk/src/java/org/apache/lucene/document/Field.java 2005-12-07 22:02:41.000000000 -1000
***************
*** 38,43 ****
--- 38,45 ----
// the one and only data object for all different kind of field values
private Object fieldsData = null;
+ // if fieldsData is a byte[], then this is how many bytes in it represent the field value
+ private int fieldsDataLength;
private boolean storeTermVector = false;
private boolean storeOffsetWithTermVector = false;
***************
*** 276,281 ****
--- 278,290 ----
* binaryValue() must be set. */
public byte[] binaryValue() { return fieldsData instanceof byte[] ? (byte[])fieldsData : null; }
+ /** Expert: The number of bytes in a binaryValue() that holds field data.
+ * This should only be used internally within Lucene -- see the Field method that accepts a length parameter
+ * and FieldsWriter.addDocument().
+ * For normal purposes, use binaryValue().length
+ */
+ public int binaryLength() { return fieldsDataLength; }
+
/**
* Create a field by specifying its name, value and how it will
* be saved in the index. Term vectors will not be stored in the index.
***************
*** 412,418 ****
this(name, string, store, index, token, false);
}
-
/**
* Create a stored field with binary value. Optionally the value may be compressed.
*
--- 421,426 ----
***************
*** 421,426 ****
--- 429,450 ----
* @param store How value should be stored (compressed or not.)
*/
public Field(String name, byte[] value, Store store) {
+ this(name, value, value.length, store);
+ }
+
+
+ /** Expert: Create a stored field with binary value. Optionally the value may be compressed.
+ * This version of the method supports reusing a byte[] across successive documents during
+ * indexing, thereby eliminating allocations of large field values. However, this value will be
+ * retained until the document is added to the index and so the same buffer cannot be used
+ * with different values on more than one field of a document.
+ *
+ * @param name The name of the field
+ * @param value The byte[] holding the binary value
+ * @param length The number of bytes in value holding data
+ * @param store How value should be stored (compressed or not.)
+ */
+ public Field(String name, byte[] value, int length, Store store) {
if (name == null)
throw new IllegalArgumentException("name cannot be null");
if (value == null)
***************
*** 428,433 ****
--- 452,458 ----
this.name = name.intern();
this.fieldsData = value;
+ this.fieldsDataLength = length;
if (store == Store.YES){
this.isStored = true;
diff -crX exclude.files LuceneTrunkFreshPull/src/java/org/apache/lucene/index/FieldsWriter.java LuceneTrunk/src/java/org/apache/lucene/index/FieldsWriter.java
*** LuceneTrunkFreshPull/src/java/org/apache/lucene/index/FieldsWriter.java 2005-12-07 21:29:59.000000000 -1000
--- LuceneTrunk/src/java/org/apache/lucene/index/FieldsWriter.java 2005-12-07 21:27:46.000000000 -1000
***************
*** 82,88 ****
byte[] data = null;
// check if it is a binary field
if (field.isBinary()) {
! data = compress(field.binaryValue());
}
else {
data = compress(field.stringValue().getBytes("UTF-8"));
--- 82,88 ----
byte[] data = null;
// check if it is a binary field
if (field.isBinary()) {
! data = compress(field.binaryValue(), field.binaryLength());
}
else {
data = compress(field.stringValue().getBytes("UTF-8"));
***************
*** 95,101 ****
// compression is disabled for the current field
if (field.isBinary()) {
byte[] data = field.binaryValue();
! final int len = data.length;
fieldsStream.writeVInt(len);
fieldsStream.writeBytes(data, len);
}
--- 95,101 ----
// compression is disabled for the current field
if (field.isBinary()) {
byte[] data = field.binaryValue();
! final int len = field.binaryLength();
fieldsStream.writeVInt(len);
fieldsStream.writeBytes(data, len);
}
***************
*** 108,120 ****
}
private final byte[] compress (byte[] input) {
// Create the compressor with highest level of compression
Deflater compressor = new Deflater();
compressor.setLevel(Deflater.BEST_COMPRESSION);
// Give the compressor the data to compress
! compressor.setInput(input);
compressor.finish();
/*
--- 108,124 ----
}
private final byte[] compress (byte[] input) {
+ return compress(input, input.length);
+ }
+
+ private final byte[] compress(byte[] input, int length) {
// Create the compressor with highest level of compression
Deflater compressor = new Deflater();
compressor.setLevel(Deflater.BEST_COMPRESSION);
// Give the compressor the data to compress
! compressor.setInput(input, 0, length);
compressor.finish();
/*
***************
*** 123,129 ****
* there is no guarantee that the compressed data will be smaller than
* the uncompressed data.
*/
! ByteArrayOutputStream bos = new ByteArrayOutputStream(input.length);
// Compress the data
byte[] buf = new byte[1024];
--- 127,133 ----
* there is no guarantee that the compressed data will be smaller than
* the uncompressed data.
*/
! ByteArrayOutputStream bos = new ByteArrayOutputStream(length);
// Compress the data
byte[] buf = new byte[1024];
diff -crX exclude.files LuceneTrunkFreshPull/src/test/org/apache/lucene/document/TestBinaryDocument.java LuceneTrunk/src/test/org/apache/lucene/document/TestBinaryDocument.java
*** LuceneTrunkFreshPull/src/test/org/apache/lucene/document/TestBinaryDocument.java 2005-12-07 21:29:48.000000000 -1000
--- LuceneTrunk/src/test/org/apache/lucene/document/TestBinaryDocument.java 2005-12-13 19:34:21.000000000 -1000
***************
*** 35,40 ****
--- 35,45 ----
String binaryValStored = "this text will be stored as a byte array in the index";
String binaryValCompressed = "this text will be also stored and compressed as a byte array in the index";
+ byte[] bufferedVal = {(byte)43, (byte)187, (byte)16, (byte)2, (byte)255, (byte)0, (byte)97, (byte)88, (byte)171, (byte)101, (byte)63, (byte)19};
+ int bufferedLength = 8;
+ byte[] bufferedCompressedVal = {(byte)1, (byte)2, (byte)3, (byte)251, (byte)252, (byte)253, (byte)255, (byte)254, (byte)128, (byte)127, (byte)13, (byte)19, (byte)181};
+ int bufferedCompressedLength = 10;
+
public void testBinaryFieldInIndex()
throws Exception
{
***************
*** 42,47 ****
--- 47,54 ----
Field binaryFldCompressed = new Field("binaryCompressed", binaryValCompressed.getBytes(), Field.Store.COMPRESS);
Field stringFldStored = new Field("stringStored", binaryValStored, Field.Store.YES, Field.Index.NO, Field.TermVector.NO);
Field stringFldCompressed = new Field("stringCompressed", binaryValCompressed, Field.Store.COMPRESS, Field.Index.NO, Field.TermVector.NO);
+ Field bufferedFld = new Field("buffered", bufferedVal, bufferedLength, Field.Store.YES);
+ Field bufferedCompressedFld = new Field ("bufferedCompressed", bufferedCompressedVal, bufferedCompressedLength, Field.Store.COMPRESS);
try {
// binary fields with store off are not allowed
***************
*** 60,67 ****
doc.add(stringFldStored);
doc.add(stringFldCompressed);
/** test for field count */
! assertEquals(4, doc.fields.size());
/** add the doc to a ram index */
RAMDirectory dir = new RAMDirectory();
--- 67,77 ----
doc.add(stringFldStored);
doc.add(stringFldCompressed);
+ doc.add(bufferedFld);
+ doc.add(bufferedCompressedFld);
+
/** test for field count */
! assertEquals(6, doc.fields.size());
/** add the doc to a ram index */
RAMDirectory dir = new RAMDirectory();
***************
*** 90,95 ****
--- 100,119 ----
String stringFldCompressedTest = new String(docFromReader.get("stringCompressed"));
assertTrue(stringFldCompressedTest.equals(binaryValCompressed));
+ byte[] bufferedFldTest = docFromReader.getBinaryValue("buffered");
+ assertTrue(bufferedFldTest.length==bufferedLength);
+ boolean bufferedSame = true;
+ for (int i=0; i