* This format has no memory overhead and requires at most 1 disk seek to - * locate a document in the fields data file. Use this format in + * locate a document in the fields data file. Use this fields index in * memory-constrained environments. */ DISK_DOC(0) { @Override - Writer newWriter(IndexOutput out) { + Writer newWriter(IndexOutput out, long startPointerBase) { return new DiskDocFieldsIndexWriter(out); } @Override @@ -57,38 +58,18 @@ }, /** - * For every document in the segment, this format stores the offset of the - * compressed chunk that contains it in the fields data file. - *
- * This fields index format requires at most 8 * numDocs bytes
- * of memory. Locating a document in the fields data file requires no disk
- * seek. Use this format when blocks are very likely to contain few
- * documents (in particular when chunkSize = 1).
- */
- MEMORY_DOC(1) {
- @Override
- Writer newWriter(IndexOutput out) throws IOException {
- return new ChunksFieldsIndexWriter(out);
- }
- @Override
- Reader newReader(IndexInput in, SegmentInfo si) throws IOException {
- return new MemoryDocFieldsIndexReader(in, si);
- }
- },
-
- /**
- * For every chunk of compressed documents, this format stores the first doc
+ * For every chunk of compressed documents, this index stores the first doc
* ID of the chunk as well as the start offset of the chunk.
*
- * This fields index format require at most
- * 12 * numChunks bytes of memory. Locating a document in the
- * fields data file requires no disk seek. Use this format when chunks are
- * likely to contain several documents.
+ * This fields index uses a very compact in-memory representation (up to
+ * 12 * numChunks bytes, but likely much less) and requires no
+ * disk seek to locate a document in the fields data file. Unless you are
+ * working with very little memory, you should use this instance.
*/
- MEMORY_CHUNK(2) {
+ MEMORY_CHUNK(1) {
@Override
- Writer newWriter(IndexOutput out) throws IOException {
- return new ChunksFieldsIndexWriter(out);
+ Writer newWriter(IndexOutput out, long startPointerBase) throws IOException {
+ return new MemoryChunkFieldsIndexWriter(out, startPointerBase);
}
@Override
Reader newReader(IndexInput in, SegmentInfo si) throws IOException {
@@ -124,7 +105,7 @@
return id;
}
- abstract Writer newWriter(IndexOutput out) throws IOException;
+ abstract Writer newWriter(IndexOutput out, long startPointerBase) throws IOException;
abstract Reader newReader(IndexInput in, SegmentInfo si) throws IOException;
@@ -176,45 +157,129 @@
}
- private static class ChunksFieldsIndexWriter extends Writer {
+ private static class MemoryChunkFieldsIndexWriter extends Writer {
+ static final int BLOCK_SIZE = 1024; // number of chunks to serialize at once
+
+ static int bitsRequired(int[] arr, int length) {
+ int or = 0;
+ for (int i = 0; i < length; ++i) {
+ or |= arr[i];
+ }
+ return PackedInts.bitsRequired(or);
+ }
+
+ static int bitsRequired(long[] arr, int length) {
+ long or = 0;
+ for (int i = 0; i < length; ++i) {
+ or |= arr[i];
+ }
+ return PackedInts.bitsRequired(or);
+ }
+
+ int numDocs;
int numChunks;
+ long firstStartPointer;
long maxStartPointer;
- GrowableWriter docBaseDeltas;
- GrowableWriter startPointerDeltas;
+ final int[] docBaseDeltas;
+ final long[] startPointerDeltas;
- ChunksFieldsIndexWriter(IndexOutput indexOutput) {
+ MemoryChunkFieldsIndexWriter(IndexOutput indexOutput, long startPointerBase) throws IOException {
super(indexOutput);
+ reset();
+ docBaseDeltas = new int[BLOCK_SIZE];
+ startPointerDeltas = new long[BLOCK_SIZE];
+ firstStartPointer = maxStartPointer = startPointerBase;
+ fieldsIndexOut.writeVInt(PackedInts.VERSION_CURRENT);
+ fieldsIndexOut.writeVLong(startPointerBase);
+ }
+
+ private void reset() {
numChunks = 0;
- maxStartPointer = 0;
- docBaseDeltas = new GrowableWriter(2, 128, PackedInts.COMPACT);
- startPointerDeltas = new GrowableWriter(5, 128, PackedInts.COMPACT);
+ numDocs = 0;
+ firstStartPointer = -1; // means unset
}
+ private void writeBlock() throws IOException {
+ assert numChunks > 0;
+ fieldsIndexOut.writeVInt(numChunks);
+
+ // doc bases
+ final int avgChunkDocs;
+ if (numChunks == 1) {
+ avgChunkDocs = 0;
+ } else {
+ avgChunkDocs = Math.round((float) (numDocs - docBaseDeltas[numChunks - 1]) / (numChunks - 1));
+ }
+ fieldsIndexOut.writeVInt(avgChunkDocs);
+ int docBase = 0;
+ long maxDelta = 0;
+ for (int i = 0; i < numChunks; ++i) {
+ final int delta = docBase - avgChunkDocs * i;
+ maxDelta = Math.max(Math.abs(delta), maxDelta);
+ docBase += docBaseDeltas[i];
+ }
+ fieldsIndexOut.writeVInt(PackedInts.bitsRequired(maxDelta));
+
+ fieldsIndexOut.writeVInt(bitsRequired(docBaseDeltas, numChunks));
+ PackedInts.Writer writer = PackedInts.getWriterNoHeader(fieldsIndexOut,
+ PackedInts.Format.PACKED, numChunks, bitsRequired(docBaseDeltas, numChunks), 1);
+ for (int i = 0; i < numChunks; ++i) {
+ writer.add(docBaseDeltas[i]);
+ }
+ writer.finish();
+
+ // start pointers
+ final long avgChunkSize;
+ if (numChunks == 1) {
+ avgChunkSize = 0;
+ } else {
+ avgChunkSize = (maxStartPointer - firstStartPointer) / (numChunks - 1);
+ }
+ fieldsIndexOut.writeVLong(avgChunkSize);
+ long startPointer = 0;
+ maxDelta = 0;
+ for (int i = 0; i < numChunks; ++i) {
+ startPointer += startPointerDeltas[i];
+ final long delta = startPointer - avgChunkSize * i;
+ maxDelta = Math.max(Math.abs(delta), maxDelta);
+ }
+ fieldsIndexOut.writeVInt(PackedInts.bitsRequired(maxDelta));
+
+ fieldsIndexOut.writeVInt(bitsRequired(startPointerDeltas, numChunks));
+ writer = PackedInts.getWriterNoHeader(fieldsIndexOut, PackedInts.Format.PACKED,
+ numChunks, bitsRequired(startPointerDeltas, numChunks), 1);
+ for (int i = 0; i < numChunks; ++i) {
+ writer.add(startPointerDeltas[i]);
+ }
+ writer.finish();
+ }
+
@Override
void writeIndex(int numDocs, long startPointer) throws IOException {
- if (numChunks == docBaseDeltas.size()) {
- final int newSize = ArrayUtil.oversize(numChunks + 1, 1);
- docBaseDeltas = docBaseDeltas.resize(newSize);
- startPointerDeltas = startPointerDeltas.resize(newSize);
+ if (firstStartPointer == -1) {
+ firstStartPointer = startPointer;
}
- docBaseDeltas.set(numChunks, numDocs);
- startPointerDeltas.set(numChunks, startPointer - maxStartPointer);
+ assert firstStartPointer > 0 && startPointer >= firstStartPointer;
+ if (numChunks == BLOCK_SIZE) {
+ writeBlock();
+ reset();
+ }
+ docBaseDeltas[numChunks] = numDocs;
+ startPointerDeltas[numChunks] = startPointer - maxStartPointer;
+
++numChunks;
+ this.numDocs += numDocs;
maxStartPointer = startPointer;
}
@Override
void finish(int numDocs) throws IOException {
- if (numChunks != docBaseDeltas.size()) {
- docBaseDeltas = docBaseDeltas.resize(numChunks);
- startPointerDeltas = startPointerDeltas.resize(numChunks);
+ if (numChunks > 0) {
+ writeBlock();
}
- fieldsIndexOut.writeVInt(numChunks);
- fieldsIndexOut.writeByte((byte) PackedInts.bitsRequired(maxStartPointer));
- docBaseDeltas.save(fieldsIndexOut);
- startPointerDeltas.save(fieldsIndexOut);
+ fieldsIndexOut.writeVInt(0); // end marker
}
}
@@ -231,9 +296,7 @@
abstract long getStartPointer(int docID) throws IOException;
public void close() throws IOException {
- if (fieldsIndexIn != null) {
- fieldsIndexIn.close();
- }
+ IOUtils.close(fieldsIndexIn);
}
public abstract Reader clone();
@@ -271,138 +334,175 @@
}
- private static class MemoryDocFieldsIndexReader extends Reader {
+ private static class MemoryChunkFieldsIndexReader extends Reader {
- private final PackedInts.Reader startPointers;
-
- MemoryDocFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
- super(fieldsIndexIn);
- final int numChunks = fieldsIndexIn.readVInt();
- final int bitsPerStartPointer = fieldsIndexIn.readByte() & 0xFF;
- if (bitsPerStartPointer > 64) {
- throw new CorruptIndexException("Corrupted");
- }
-
- final PackedInts.Reader chunkDocs = PackedInts.getReader(fieldsIndexIn);
- if (chunkDocs.size() != numChunks) {
- throw new CorruptIndexException("Expected " + numChunks + " chunks, but got " + chunkDocs.size());
- }
-
- final PackedInts.ReaderIterator startPointerDeltas = PackedInts.getReaderIterator(fieldsIndexIn, PackedInts.DEFAULT_BUFFER_SIZE);
- if (startPointerDeltas.size() != numChunks) {
- throw new CorruptIndexException("Expected " + numChunks + " chunks, but got " + startPointerDeltas.size());
- }
- final PackedInts.Mutable startPointers = PackedInts.getMutable(si.getDocCount(), bitsPerStartPointer, PackedInts.COMPACT);
- int docID = 0;
- long startPointer = 0;
- for (int i = 0; i < numChunks; ++i) {
- startPointer += startPointerDeltas.next();
- final int chunkDocCount = (int) chunkDocs.get(i);
- for (int j = 0; j < chunkDocCount; ++j) {
- startPointers.set(docID++, startPointer);
- }
- }
- if (docID != si.getDocCount()) {
- throw new CorruptIndexException("Expected " + si.getDocCount() + " docs, got " + docID);
- }
-
- this.startPointers = startPointers;
+ static long moveLowOrderBitToSign(long n) {
+ return ((n >>> 1) ^ -(n & 1));
}
- private MemoryDocFieldsIndexReader(PackedInts.Reader startPointers) {
- super(null);
- this.startPointers = startPointers;
+ static long moveSignToLowOrderBit(long n) {
+ return (n >> 63) ^ (n << 1);
}
- @Override
- long getStartPointer(int docID) throws IOException {
- return startPointers.get(docID);
- }
+ private final int maxDoc;
+ private final int[] docBases;
+ private final long[] startPointers;
+ private final int[] avgChunkDocs;
+ private final long[] avgChunkSizes;
+ private final PackedInts.Reader[] docBasesDeltas; // delta from the avg
+ private final PackedInts.Reader[] startPointersDeltas; // delta from the avg
- @Override
- public Reader clone() {
- if (fieldsIndexIn == null) {
- return this;
- } else {
- return new MemoryDocFieldsIndexReader(startPointers);
- }
- }
+ MemoryChunkFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
+ super(fieldsIndexIn);
+ maxDoc = si.getDocCount();
+ int[] docBases = new int[16];
+ long[] startPointers = new long[16];
+ int[] avgChunkDocs = new int[16];
+ long[] avgChunkSizes = new long[16];
+ PackedInts.Reader[] docBasesDeltas = new PackedInts.Reader[16];
+ PackedInts.Reader[] startPointersDeltas = new PackedInts.Reader[16];
- }
+ final int packedIntsVersion = fieldsIndexIn.readVInt();
- private static class MemoryChunkFieldsIndexReader extends Reader {
+ int blockCount = 0;
+ int docBase = 0;
+ long startPointer = fieldsIndexIn.readVLong();
- private final PackedInts.Reader docBases;
- private final PackedInts.Reader startPointers;
+ for (;;) {
+ final int numChunks = fieldsIndexIn.readVInt();
+ if (numChunks == 0) {
+ break;
+ }
+ if (blockCount == docBases.length) {
+ final int newSize = ArrayUtil.oversize(blockCount + 1, 8);
+ docBases = Arrays.copyOf(docBases, newSize);
+ startPointers = Arrays.copyOf(startPointers, newSize);
+ avgChunkDocs = Arrays.copyOf(avgChunkDocs, newSize);
+ avgChunkSizes = Arrays.copyOf(avgChunkSizes, newSize);
+ docBasesDeltas = Arrays.copyOf(docBasesDeltas, newSize);
+ startPointersDeltas = Arrays.copyOf(startPointersDeltas, newSize);
+ }
- MemoryChunkFieldsIndexReader(IndexInput fieldsIndexIn, SegmentInfo si) throws IOException {
- super(fieldsIndexIn);
- final int numChunks = fieldsIndexIn.readVInt();
- final int bitsPerStartPointer = fieldsIndexIn.readByte() & 0xFF;
- if (bitsPerStartPointer > 64) {
- throw new CorruptIndexException("Corrupted");
- }
+ // doc bases
+ avgChunkDocs[blockCount] = fieldsIndexIn.readVInt();
+ final int bitsPerDocBase = 1 + fieldsIndexIn.readVInt();
+ if (bitsPerDocBase > 32) {
+ throw new CorruptIndexException("Corrupted");
+ }
+ // deltas from average
+ final PackedInts.Mutable dbDeltas = PackedInts.getMutable(numChunks, bitsPerDocBase, PackedInts.COMPACT);
+ // deltas from the previous value
+ final PackedInts.ReaderIterator docBasesDiffs = PackedInts.getReaderIteratorNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, fieldsIndexIn.readVInt(), 1);
+ docBases[blockCount] = docBase;
+ for (int i = 0; i < numChunks; ++i) {
+ final long delta = docBase - docBases[blockCount] - avgChunkDocs[blockCount] * i;
+ dbDeltas.set(i, moveSignToLowOrderBit(delta));
+ docBase += docBasesDiffs.next();
+ }
+ docBasesDeltas[blockCount] = dbDeltas;
- final PackedInts.ReaderIterator docBaseDeltas = PackedInts.getReaderIterator(fieldsIndexIn, PackedInts.DEFAULT_BUFFER_SIZE);
- if (docBaseDeltas.size() != numChunks) {
- throw new CorruptIndexException("Expected " + numChunks + " chunks, but got " + docBaseDeltas.size());
- }
- final PackedInts.Mutable docBases = PackedInts.getMutable(numChunks, PackedInts.bitsRequired(Math.max(0, si.getDocCount() - 1)), PackedInts.COMPACT);
+ // start pointers
+ avgChunkSizes[blockCount] = fieldsIndexIn.readVInt();
+ final int bitsPerStartPointer = 1 + fieldsIndexIn.readVInt();
+ if (bitsPerDocBase > 64) {
+ throw new CorruptIndexException("Corrupted");
+ }
+ // deltas from average
+ final PackedInts.Mutable spDeltas = PackedInts.getMutable(numChunks, bitsPerStartPointer, PackedInts.COMPACT);
+ // deltas from the previous value
+ final PackedInts.ReaderIterator startPointersDiffs = PackedInts.getReaderIteratorNoHeader(fieldsIndexIn, PackedInts.Format.PACKED, packedIntsVersion, numChunks, fieldsIndexIn.readVInt(), 1);
+ for (int i = 0; i < numChunks; ++i) {
+ startPointer += startPointersDiffs.next();
+ if (i == 0) {
+ startPointers[blockCount] = startPointer;
+ }
+ final long delta = startPointer - startPointers[blockCount] - avgChunkSizes[blockCount] * i;
+ spDeltas.set(i, moveSignToLowOrderBit(delta));
+ }
+ startPointersDeltas[blockCount] = spDeltas;
- int docBase = 0;
- for (int i = 0; i < numChunks; ++i) {
- docBases.set(i, docBase);
- docBase += docBaseDeltas.next();
+ ++blockCount;
}
- if (docBase != si.getDocCount()) {
- throw new CorruptIndexException("Expected " + si.getDocCount() + " docs, got " + docBase);
- }
- final PackedInts.ReaderIterator startPointerDeltas = PackedInts.getReaderIterator(fieldsIndexIn, PackedInts.DEFAULT_BUFFER_SIZE);
- if (startPointerDeltas.size() != numChunks) {
- throw new CorruptIndexException("Expected " + numChunks + " chunks, but got " + startPointerDeltas.size());
- }
- final PackedInts.Mutable startPointers = PackedInts.getMutable(numChunks, bitsPerStartPointer, PackedInts.COMPACT);
- long startPointer = 0;
- for (int i = 0; i < numChunks; ++i) {
- startPointer += startPointerDeltas.next();
- startPointers.set(i, startPointer);
- }
-
- this.docBases = docBases;
- this.startPointers = startPointers;
+ this.docBases = Arrays.copyOf(docBases, blockCount);
+ this.startPointers = Arrays.copyOf(startPointers, blockCount);
+ this.avgChunkDocs = avgChunkDocs;
+ this.avgChunkSizes = avgChunkSizes;
+ this.docBasesDeltas = Arrays.copyOf(docBasesDeltas, blockCount);
+ this.startPointersDeltas = Arrays.copyOf(startPointersDeltas, blockCount);
}
- private MemoryChunkFieldsIndexReader(PackedInts.Reader docBases, PackedInts.Reader startPointers) {
+ private MemoryChunkFieldsIndexReader(MemoryChunkFieldsIndexReader other) {
super(null);
- this.docBases = docBases;
- this.startPointers = startPointers;
+ this.maxDoc = other.maxDoc;
+ this.docBases = other.docBases;
+ this.startPointers = other.startPointers;
+ this.avgChunkDocs = other.avgChunkDocs;
+ this.avgChunkSizes = other.avgChunkSizes;
+ this.docBasesDeltas = other.docBasesDeltas;
+ this.startPointersDeltas = other.startPointersDeltas;
}
- @Override
- long getStartPointer(int docID) {
- assert docBases.size() > 0;
- int lo = 0, hi = docBases.size() - 1;
+ private int block(int docID) {
+ int lo = 0, hi = docBases.length - 1;
while (lo <= hi) {
final int mid = (lo + hi) >>> 1;
- final long midValue = docBases.get(mid);
+ final int midValue = docBases[mid];
if (midValue == docID) {
- return startPointers.get(mid);
+ return mid;
} else if (midValue < docID) {
lo = mid + 1;
} else {
hi = mid - 1;
}
}
- return startPointers.get(hi);
+ return hi;
}
+ private int relativeDocBase(int block, int relativeChunk) {
+ final int expected = avgChunkDocs[block] * relativeChunk;
+ final long delta = moveLowOrderBitToSign(docBasesDeltas[block].get(relativeChunk));
+ return expected + (int) delta;
+ }
+
+ private long relativeStartPointer(int block, int relativeChunk) {
+ final long expected = avgChunkSizes[block] * relativeChunk;
+ final long delta = moveLowOrderBitToSign(startPointersDeltas[block].get(relativeChunk));
+ return expected + delta;
+ }
+
+ private int relativeChunk(int block, int relativeDoc) {
+ int lo = 0, hi = docBasesDeltas[block].size() - 1;
+ while (lo <= hi) {
+ final int mid = (lo + hi) >>> 1;
+ final int midValue = relativeDocBase(block, mid);
+ if (midValue == relativeDoc) {
+ return mid;
+ } else if (midValue < relativeDoc) {
+ lo = mid + 1;
+ } else {
+ hi = mid - 1;
+ }
+ }
+ return hi;
+ }
+
@Override
+ long getStartPointer(int docID) {
+ if (docID < 0 || docID >= maxDoc) {
+ throw new IllegalArgumentException("docID out of range [0-" + maxDoc + "]: " + docID);
+ }
+ final int block = block(docID);
+ final int relativeChunk = relativeChunk(block, docID - docBases[block]);
+ return startPointers[block] + relativeStartPointer(block, relativeChunk);
+ }
+
+ @Override
public Reader clone() {
if (fieldsIndexIn == null) {
return this;
} else {
- return new MemoryChunkFieldsIndexReader(docBases, startPointers);
+ return new MemoryChunkFieldsIndexReader(this);
}
}
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (révision 1403708)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsFormat.java (copie de travail)
@@ -90,9 +90,7 @@
* @see CompressingStoredFieldsFormat#CompressingStoredFieldsFormat(CompressionMode, int, CompressingStoredFieldsIndex)
*/
public CompressingStoredFieldsFormat(CompressionMode compressionMode, int chunkSize) {
- this (compressionMode, chunkSize, chunkSize == 1
- ? CompressingStoredFieldsIndex.MEMORY_DOC
- : CompressingStoredFieldsIndex.MEMORY_CHUNK);
+ this (compressionMode, chunkSize, CompressingStoredFieldsIndex.MEMORY_CHUNK);
}
/**
Index: lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
===================================================================
--- lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (révision 1403708)
+++ lucene/codecs/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (copie de travail)
@@ -101,13 +101,13 @@
assert HEADER_LENGTH_IDX == indexStream.getFilePointer();
assert HEADER_LENGTH_DAT == fieldsStream.getFilePointer();
+ fieldsStream.writeVInt(PackedInts.VERSION_CURRENT);
+ fieldsStream.writeVInt(compressionMode.getId());
+
indexStream.writeVInt(storedFieldsIndex.getId());
- indexWriter = storedFieldsIndex.newWriter(indexStream);
+ indexWriter = storedFieldsIndex.newWriter(indexStream, fieldsStream.getFilePointer());
indexStream = null;
- fieldsStream.writeVInt(PackedInts.VERSION_CURRENT);
- fieldsStream.writeVInt(compressionMode.getId());
-
success = true;
} finally {
if (!success) {