You should use this class instead of {@link AppendingLongBuffer} only when
+ * you need random write-access. Otherwise this class will likely be slower and
+ * less memory-efficient.
+ * @lucene.internal
+ */
+public final class PagedGrowableWriter {
+
+ private final long size;
+ private final int pageShift;
+ private final int pageMask;
+ private final GrowableWriter[] subWriters;
+ private final int startBitsPerValue;
+ private final float acceptableOverheadRatio;
+
+ /**
+ * Create a new {@link PagedGrowableWriter} instance.
+ *
+ * @param size the number of values to store.
+ * @param pageSize the number of values per page
+ * @param startBitsPerValue the initial number of bits per value
+ * @param acceptableOverheadRatio an acceptable overhead ratio
+ */
+ public PagedGrowableWriter(long size, int pageSize,
+ int startBitsPerValue, float acceptableOverheadRatio) {
+ this(size, pageSize, startBitsPerValue, acceptableOverheadRatio, true);
+ }
+
+ PagedGrowableWriter(long size, int pageSize,int startBitsPerValue, float acceptableOverheadRatio, boolean fillPages) {
+ super();
+ this.size = size;
+ this.startBitsPerValue = startBitsPerValue;
+ this.acceptableOverheadRatio = acceptableOverheadRatio;
+ if (pageSize < 64 || ((pageSize & (pageSize - 1)) != 0)) {
+ throw new IllegalArgumentException("pageSize must be >= 64 and a power of 2, got " + pageSize);
+ }
+ pageShift = 31 - Integer.numberOfLeadingZeros(pageSize);
+ assert (1 << pageShift) == pageSize;
+ pageMask = pageSize - 1;
+ final int numPages = (int) ((size + pageSize - 1) / pageSize);
+ if ((long) numPages * pageSize < size || (long) (numPages - 1) * pageSize > size) {
+ throw new IllegalArgumentException("pageSize must be chosen so that there are at most Integer.MAX_VALUE pages, got size=" + size + ", pageSize=" + pageSize);
+ }
+ subWriters = new GrowableWriter[numPages];
+ if (fillPages) {
+ for (int i = 0; i < numPages; ++i) {
+ subWriters[i] = new GrowableWriter(startBitsPerValue, pageSize, acceptableOverheadRatio);
+ }
+ }
+ }
+
+ private int pageSize() {
+ return pageMask + 1;
+ }
+
+ /** The number of values. */
+ public long size() {
+ return size;
+ }
+
+ int pageIndex(long index) {
+ return (int) (index >>> pageShift);
+ }
+
+ int indexInPage(long index) {
+ return (int) index & pageMask;
+ }
+
+ /** Get value at index. */
+ public long get(long index) {
+ assert index >= 0 && index < size;
+ final int pageIndex = pageIndex(index);
+ final int indexInPage = indexInPage(index);
+ return subWriters[pageIndex].get(indexInPage);
+ }
+
+ /** Set value at index. */
+ public void set(long index, long value) {
+ assert index >= 0 && index < size;
+ final int pageIndex = pageIndex(index);
+ final int indexInPage = indexInPage(index);
+ subWriters[pageIndex].set(indexInPage, value);
+ }
+
+ /** Create a new {@link PagedGrowableWriter} of size newSize
+ * based on the content of this buffer. This method is much more efficient
+ * than creating a new {@link PagedGrowableWriter} and copying values one by
+ * one. */
+ public PagedGrowableWriter resize(long newSize) {
+ PagedGrowableWriter newWriter = new PagedGrowableWriter(newSize, pageSize(), startBitsPerValue, acceptableOverheadRatio, false);
+ final int numCommonPages = Math.min(newWriter.subWriters.length, subWriters.length);
+ final long[] copyBuffer = new long[1024];
+ for (int i = 0; i < numCommonPages; ++i) {
+ final int bpv = subWriters[i].getBitsPerValue();
+ newWriter.subWriters[i] = new GrowableWriter(bpv, pageSize(), acceptableOverheadRatio);
+ PackedInts.copy(subWriters[i], 0, newWriter.subWriters[i], 0, pageSize(), copyBuffer);
+ }
+ for (int i = numCommonPages; i < newWriter.subWriters.length; ++i) {
+ newWriter.subWriters[i] = new GrowableWriter(startBitsPerValue, pageSize(), acceptableOverheadRatio);
+ }
+ return newWriter;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName() + "(size=" + size() + ",pageSize=" + (pageMask+1) + ")";
+ }
+
+}
Property changes on: lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java
___________________________________________________________________
Added: svn:eol-style
## -0,0 +1 ##
+native
\ No newline at end of property
Index: lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
===================================================================
--- lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (revision 1488122)
+++ lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (working copy)
@@ -659,6 +659,61 @@
assertEquals(1 << 10, wrt.get(valueCount - 1));
}
+ public void testPagedGrowableWriter() {
+ int pageSize = 1 << (_TestUtil.nextInt(random(), 6, 22));
+ // supports 0 values?
+ PagedGrowableWriter writer = new PagedGrowableWriter(0, pageSize, _TestUtil.nextInt(random(), 1, 64), random().nextFloat());
+ assertEquals(0, writer.size());
+
+ // compare against AppendingLongBuffer
+ AppendingLongBuffer buf = new AppendingLongBuffer();
+ int size = random().nextInt(1000000);
+ long max = 5;
+ for (int i = 0; i < size; ++i) {
+ buf.add(_TestUtil.nextLong(random(), 0, max));
+ if (rarely()) {
+ max = PackedInts.maxValue(rarely() ? _TestUtil.nextInt(random(), 0, 63) : _TestUtil.nextInt(random(), 0, 31));
+ }
+ }
+ writer = new PagedGrowableWriter(size, pageSize, _TestUtil.nextInt(random(), 1, 64), random().nextFloat());
+ assertEquals(size, writer.size());
+ for (int i = size - 1; i >= 0; --i) {
+ writer.set(i, buf.get(i));
+ }
+ for (int i = 0; i < size; ++i) {
+ assertEquals(buf.get(i), writer.get(i));
+ }
+
+ // test copy
+ PagedGrowableWriter copy = writer.resize(_TestUtil.nextLong(random(), writer.size(), 2 * writer.size()));
+ for (long i = 0; i < copy.size(); ++i) {
+ if (i < writer.size()) {
+ assertEquals(writer.get(i), copy.get(i));
+ } else {
+ assertEquals(0, copy.get(i));
+ }
+ }
+ }
+
+ // memory hole
+ @Ignore
+ public void testPagedGrowableWriterOverflow() {
+ final long size = _TestUtil.nextLong(random(), 2 * (long) Integer.MAX_VALUE, 3 * (long) Integer.MAX_VALUE);
+ final int pageSize = 1 << (_TestUtil.nextInt(random(), 16, 24));
+ final PagedGrowableWriter writer = new PagedGrowableWriter(size, pageSize, 1, random().nextFloat());
+ final long index = _TestUtil.nextLong(random(), (long) Integer.MAX_VALUE, size);
+ writer.set(index, 2);
+ assertEquals(2, writer.get(index));
+ for (int i = 0; i < 1000000; ++i) {
+ final long idx = _TestUtil.nextLong(random(), 0, size);
+ if (idx == index) {
+ assertEquals(2, writer.get(idx));
+ } else {
+ assertEquals(0, writer.get(idx));
+ }
+ }
+ }
+
public void testSave() throws IOException {
final int valueCount = _TestUtil.nextInt(random(), 1, 2048);
for (int bpv = 1; bpv <= 64; ++bpv) {