Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1488278) +++ lucene/CHANGES.txt (working copy) @@ -169,6 +169,10 @@ * LUCENE-5022: Added FacetResult.mergeHierarchies to merge multiple FacetResult of the same dimension into a single one with the reconstructed hierarchy. (Shai Erera) + +* LUCENE-5026: Added PagedGrowableWriter, a new internal packed-ints structure + that grows on demand, can store more than 2B values and supports random write + and read access. (Adrien Grand) Build Index: lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (revision 1488278) +++ lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (working copy) @@ -659,6 +659,61 @@ assertEquals(1 << 10, wrt.get(valueCount - 1)); } + public void testPagedGrowableWriter() { + int pageSize = 1 << (_TestUtil.nextInt(random(), 6, 22)); + // supports 0 values? + PagedGrowableWriter writer = new PagedGrowableWriter(0, pageSize, _TestUtil.nextInt(random(), 1, 64), random().nextFloat()); + assertEquals(0, writer.size()); + + // compare against AppendingLongBuffer + AppendingLongBuffer buf = new AppendingLongBuffer(); + int size = random().nextInt(1000000); + long max = 5; + for (int i = 0; i < size; ++i) { + buf.add(_TestUtil.nextLong(random(), 0, max)); + if (rarely()) { + max = PackedInts.maxValue(rarely() ? _TestUtil.nextInt(random(), 0, 63) : _TestUtil.nextInt(random(), 0, 31)); + } + } + writer = new PagedGrowableWriter(size, pageSize, _TestUtil.nextInt(random(), 1, 64), random().nextFloat()); + assertEquals(size, writer.size()); + for (int i = size - 1; i >= 0; --i) { + writer.set(i, buf.get(i)); + } + for (int i = 0; i < size; ++i) { + assertEquals(buf.get(i), writer.get(i)); + } + + // test copy + PagedGrowableWriter copy = writer.resize(_TestUtil.nextLong(random(), writer.size(), 2 * writer.size())); + for (long i = 0; i < copy.size(); ++i) { + if (i < writer.size()) { + assertEquals(writer.get(i), copy.get(i)); + } else { + assertEquals(0, copy.get(i)); + } + } + } + + // memory hole + @Ignore + public void testPagedGrowableWriterOverflow() { + final long size = _TestUtil.nextLong(random(), 2 * (long) Integer.MAX_VALUE, 3 * (long) Integer.MAX_VALUE); + final int pageSize = 1 << (_TestUtil.nextInt(random(), 16, 24)); + final PagedGrowableWriter writer = new PagedGrowableWriter(size, pageSize, 1, random().nextFloat()); + final long index = _TestUtil.nextLong(random(), (long) Integer.MAX_VALUE, size); + writer.set(index, 2); + assertEquals(2, writer.get(index)); + for (int i = 0; i < 1000000; ++i) { + final long idx = _TestUtil.nextLong(random(), 0, size); + if (idx == index) { + assertEquals(2, writer.get(idx)); + } else { + assertEquals(0, writer.get(idx)); + } + } + } + public void testSave() throws IOException { final int valueCount = _TestUtil.nextInt(random(), 1, 2048); for (int bpv = 1; bpv <= 64; ++bpv) { Index: lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java (revision 1488278) +++ lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java (working copy) @@ -34,7 +34,8 @@ import org.junit.Ignore; import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; -@Ignore("Requires tons of heap to run (10G works)") +// nocommit +//@Ignore("Requires tons of heap to run (420G works)") @TimeoutSuite(millis = 100 * TimeUnits.HOUR) public class Test2BFST extends LuceneTestCase { @@ -50,12 +51,12 @@ for(int doPackIter=0;doPackIter<2;doPackIter++) { boolean doPack = doPackIter == 1; - // Build FST w/ NoOutputs and stop when nodeCount > 3B + // Build FST w/ NoOutputs and stop when nodeCount > 2.2B if (!doPack) { System.out.println("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS"); Outputs outputs = NoOutputs.getSingleton(); Object NO_OUTPUT = outputs.getNoOutput(); - final Builder b = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, false, false, Integer.MAX_VALUE, outputs, + final Builder b = new Builder(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs, null, doPack, PackedInts.COMPACT, true, 15); int count = 0; @@ -72,7 +73,7 @@ if (count % 100000 == 0) { System.out.println(count + ": " + b.fstSizeInBytes() + " bytes; " + b.getTotStateCount() + " nodes"); } - if (b.getTotStateCount() > LIMIT) { + if (b.getTotStateCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) { break; } nextInput(r, ints2); Index: lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java (revision 1488278) +++ lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java (working copy) @@ -21,19 +21,20 @@ import org.apache.lucene.util.packed.GrowableWriter; import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.util.packed.PagedGrowableWriter; // Used to dedup states (lookup already-frozen states) final class NodeHash { - private GrowableWriter table; - private int count; - private int mask; + private PagedGrowableWriter table; + private long count; + private long mask; private final FST fst; private final FST.Arc scratchArc = new FST.Arc(); private final FST.BytesReader in; public NodeHash(FST fst, FST.BytesReader in) { - table = new GrowableWriter(8, 16, PackedInts.COMPACT); + table = new PagedGrowableWriter(16, 1<<30, 8, PackedInts.COMPACT); mask = 15; this.fst = fst; this.in = in; @@ -69,10 +70,10 @@ // hash code for an unfrozen node. This must be identical // to the un-frozen case (below)!! - private int hash(Builder.UnCompiledNode node) { + private long hash(Builder.UnCompiledNode node) { final int PRIME = 31; //System.out.println("hash unfrozen"); - int h = 0; + long h = 0; // TODO: maybe if number of arcs is high we can safely subsample? for(int arcIdx=0;arcIdx arc = node.arcs[arcIdx]; @@ -87,14 +88,14 @@ } } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); - return h & Integer.MAX_VALUE; + return h & Long.MAX_VALUE; } // hash code for a frozen node - private int hash(long node) throws IOException { + private long hash(long node) throws IOException { final int PRIME = 31; //System.out.println("hash frozen node=" + node); - int h = 0; + long h = 0; fst.readFirstRealTargetArc(node, scratchArc, in); while(true) { //System.out.println(" label=" + scratchArc.label + " target=" + scratchArc.target + " h=" + h + " output=" + fst.outputs.outputToString(scratchArc.output) + " next?=" + scratchArc.flag(4) + " final?=" + scratchArc.isFinal() + " pos=" + in.getPosition()); @@ -111,13 +112,13 @@ fst.readNextRealArc(scratchArc, in); } //System.out.println(" ret " + (h&Integer.MAX_VALUE)); - return h & Integer.MAX_VALUE; + return h & Long.MAX_VALUE; } public long add(Builder.UnCompiledNode nodeIn) throws IOException { - // System.out.println("hash: add count=" + count + " vs " + table.size()); - final int h = hash(nodeIn); - int pos = h & mask; + //System.out.println("hash: add count=" + count + " vs " + table.size() + " mask=" + mask); + final long h = hash(nodeIn); + long pos = h & mask; int c = 0; while(true) { final long v = table.get(pos); @@ -128,7 +129,8 @@ assert hash(node) == h : "frozenHash=" + hash(node) + " vs h=" + h; count++; table.set(pos, node); - if (table.size() < 2*count) { + // Rehash at 2/3 occupancy: + if (count > 2*table.size()/3) { rehash(); } return node; @@ -144,7 +146,7 @@ // called only by rehash private void addNew(long address) throws IOException { - int pos = hash(address) & mask; + long pos = hash(address) & mask; int c = 0; while(true) { if (table.get(pos) == 0) { @@ -158,15 +160,12 @@ } private void rehash() throws IOException { - final GrowableWriter oldTable = table; + final PagedGrowableWriter oldTable = table; - if (oldTable.size() >= Integer.MAX_VALUE/2) { - throw new IllegalStateException("FST too large (> 2.1 GB)"); - } - - table = new GrowableWriter(oldTable.getBitsPerValue(), 2*oldTable.size(), PackedInts.COMPACT); + table = new PagedGrowableWriter(2*oldTable.size(), 1<<30, PackedInts.bitsRequired(count), PackedInts.COMPACT); + System.out.println("rehash to " + table.size()); mask = table.size()-1; - for(int idx=0;idxSame as PackedInts.Mutable but grows the number of bits per values when needed.
  • Useful to build a PackedInts.Mutable from a read-once stream of longs.
    • +
  • {@link org.apache.lucene.util.packed.PagedGrowableWriter}
      +
    • Slices data into fixed-size blocks stored in GrowableWriters.
      • +
    • Supports more than 2B values.
      • +
    • You should use AppendingLongBuffer instead if you don't need random write access.
      • +
  • {@link org.apache.lucene.util.packed.AppendingLongBuffer}
    • Can store any sequence of longs.
    • Compression is good when values are close to each other.
      • Index: lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java (working copy) @@ -0,0 +1,139 @@ +package org.apache.lucene.util.packed; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A {@link PagedGrowableWriter}. This class slices data into fixed-size blocks + * which have independent numbers of bits per value and grow on-demand. + *

      You should use this class instead of {@link AppendingLongBuffer} only when + * you need random write-access. Otherwise this class will likely be slower and + * less memory-efficient. + * @lucene.internal + */ +public final class PagedGrowableWriter { + + private final long size; + private final int pageShift; + private final int pageMask; + private final GrowableWriter[] subWriters; + private final int startBitsPerValue; + private final float acceptableOverheadRatio; + + /** + * Create a new {@link PagedGrowableWriter} instance. + * + * @param size the number of values to store. + * @param pageSize the number of values per page + * @param startBitsPerValue the initial number of bits per value + * @param acceptableOverheadRatio an acceptable overhead ratio + */ + public PagedGrowableWriter(long size, int pageSize, + int startBitsPerValue, float acceptableOverheadRatio) { + this(size, pageSize, startBitsPerValue, acceptableOverheadRatio, true); + } + + PagedGrowableWriter(long size, int pageSize,int startBitsPerValue, float acceptableOverheadRatio, boolean fillPages) { + super(); + this.size = size; + this.startBitsPerValue = startBitsPerValue; + this.acceptableOverheadRatio = acceptableOverheadRatio; + if (pageSize < 64 || ((pageSize & (pageSize - 1)) != 0)) { + throw new IllegalArgumentException("pageSize must be >= 64 and a power of 2, got " + pageSize); + } + pageShift = 31 - Integer.numberOfLeadingZeros(pageSize); + assert (1 << pageShift) == pageSize; + pageMask = pageSize - 1; + final int numPages = (int) ((size + pageSize - 1) / pageSize); + if ((long) numPages * pageSize < size || (long) (numPages - 1) * pageSize > size) { + throw new IllegalArgumentException("pageSize must be chosen so that there are at most Integer.MAX_VALUE pages, got size=" + size + ", pageSize=" + pageSize); + } + subWriters = new GrowableWriter[numPages]; + if (fillPages) { + for (int i = 0; i < numPages; ++i) { + int thisPageSize; + if (i == numPages-1) { + thisPageSize = (int) (size % pageSize); + if (thisPageSize == 0) { + thisPageSize = pageSize; + } + } else { + thisPageSize = pageSize; + } + subWriters[i] = new GrowableWriter(startBitsPerValue, thisPageSize, acceptableOverheadRatio); + } + } + } + + private int pageSize() { + return pageMask + 1; + } + + /** The number of values. */ + public long size() { + return size; + } + + int pageIndex(long index) { + return (int) (index >>> pageShift); + } + + int indexInPage(long index) { + return (int) index & pageMask; + } + + /** Get value at index. */ + public long get(long index) { + assert index >= 0 && index < size: "index=" + index + " size=" + size; + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + return subWriters[pageIndex].get(indexInPage); + } + + /** Set value at index. */ + public void set(long index, long value) { + assert index >= 0 && index < size; + final int pageIndex = pageIndex(index); + final int indexInPage = indexInPage(index); + subWriters[pageIndex].set(indexInPage, value); + } + + /** Create a new {@link PagedGrowableWriter} of size newSize + * based on the content of this buffer. This method is much more efficient + * than creating a new {@link PagedGrowableWriter} and copying values one by + * one. */ + public PagedGrowableWriter resize(long newSize) { + PagedGrowableWriter newWriter = new PagedGrowableWriter(newSize, pageSize(), startBitsPerValue, acceptableOverheadRatio, false); + final int numCommonPages = Math.min(newWriter.subWriters.length, subWriters.length); + final long[] copyBuffer = new long[1024]; + for (int i = 0; i < numCommonPages; ++i) { + final int bpv = subWriters[i].getBitsPerValue(); + newWriter.subWriters[i] = new GrowableWriter(bpv, pageSize(), acceptableOverheadRatio); + PackedInts.copy(subWriters[i], 0, newWriter.subWriters[i], 0, pageSize(), copyBuffer); + } + for (int i = numCommonPages; i < newWriter.subWriters.length; ++i) { + newWriter.subWriters[i] = new GrowableWriter(startBitsPerValue, pageSize(), acceptableOverheadRatio); + } + return newWriter; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(size=" + size() + ",pageSize=" + (pageMask+1) + ")"; + } + +} Property changes on: lucene/core/src/java/org/apache/lucene/util/packed/PagedGrowableWriter.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (revision 1488278) +++ lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (working copy) @@ -1200,31 +1200,37 @@ } } else { // use bulk operations - long[] buf = new long[Math.min(capacity, len)]; - int remaining = 0; - while (len > 0) { - final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining)); - assert read > 0; - srcPos += read; - len -= read; - remaining += read; - final int written = dest.set(destPos, buf, 0, remaining); - assert written > 0; - destPos += written; - if (written < remaining) { - System.arraycopy(buf, written, buf, 0, remaining - written); - } - remaining -= written; + final long[] buf = new long[Math.min(capacity, len)]; + copy(src, srcPos, dest, destPos, len, buf); + } + } + + /** Same as {@link #copy(Reader, int, Mutable, int, int, int)} but using a pre-allocated buffer. */ + static void copy(Reader src, int srcPos, Mutable dest, int destPos, int len, long[] buf) { + assert buf.length > 0; + int remaining = 0; + while (len > 0) { + final int read = src.get(srcPos, buf, remaining, Math.min(len, buf.length - remaining)); + assert read > 0; + srcPos += read; + len -= read; + remaining += read; + final int written = dest.set(destPos, buf, 0, remaining); + assert written > 0; + destPos += written; + if (written < remaining) { + System.arraycopy(buf, written, buf, 0, remaining - written); } - while (remaining > 0) { - final int written = dest.set(destPos, buf, 0, remaining); - destPos += written; - remaining -= written; - System.arraycopy(buf, written, buf, 0, remaining); - } + remaining -= written; } + while (remaining > 0) { + final int written = dest.set(destPos, buf, 0, remaining); + destPos += written; + remaining -= written; + System.arraycopy(buf, written, buf, 0, remaining); + } } - + /** * Expert: reads only the metadata from a stream. This is useful to later * restore a stream or open a direct reader via