Index: src/java/org/apache/lucene/util/pfor/PFor.java
===================================================================
--- src/java/org/apache/lucene/util/pfor/PFor.java (revision 0)
+++ src/java/org/apache/lucene/util/pfor/PFor.java (revision 0)
@@ -0,0 +1,583 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.IntBuffer;
+import java.util.Arrays;
+
+/** Patched Frame of Reference PFOR compression/decompression.
+ *
+ * As defined in:
+ * Super-Scalar RAM-CPU Cache Compression
+ * Marcin Zukowski, Sándor Héman, Niels Nes, Peter Boncz, 2006.
+ * with extensions from:
+ * Performance of Compressed Inverted List Caching in Search Engines
+ * Jiangong Zhang, Xiaohui Long, Torsten Suel, 2008.
+ *
+ * This class does not provide delta coding because the lucene index
+ * structures already have that.
+ *
+ * The implementation uses 0 as lower bound for the frame,
+ * so small positive integers will be most effectively compressed.
+ *
+ * Some optimized code is used for decompression,
+ * see class ForDecompress and its subclasses.
+ *
Good decompression performance will depend on the performance
+ * of java.nio.IntBuffer indexed get() methods.
+ *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux.
+ *
+ * To be done:
+ *
+ * - Try and move start point of first exception to natural boundary:
+ * short at even byte position, int at quadruple. This may speed up
+ * exception decoding and avoid overlaps between buffered ints for exceptions.
+ * Optimize exception decoding code.
+ *
-
+ * Optimize compression code.
+ *
-
+ * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression for to
+ * use the relative get() method.
+ *
-
+ * Check javadoc generation and generated javadocs. Add javadoc code references.
+ *
+ */
+public class PFor {
+ /** Number of frame bits. 2**numFrameBits - 1 is the maximum non exception value */
+ private int numFrameBits;
+
+ /** Index on input and in compressed array of first exception, -1 when no exceptions */
+ private int firstExceptionIndex;
+
+ /** Size of input, use -1 as long as no input available. */
+ private int inputSize = -1;
+
+ /** Constant header tag to allow other compression methods, use value 001 for PFor */
+ private int compressionMethod;
+ private final int PFOR_COMPRESSION = 1;
+
+ /** How to encode PFor exceptions: 0: byte, 1: short, 2:int, 3: long */
+ private int exceptionCode = -1;
+
+ /** Total number of exception values */
+ private int numExceptions;
+
+ /** int buffer for compressed data */
+ private IntBuffer intBuffer;
+
+ /** index in int buffer for header */
+ private final int HEADER_INDEX = 0;
+
+ /** Start index in int buffer of array integers each compressed to numFrameBits. */
+ private final int COMPRESSED_INDEX = HEADER_INDEX + 1;
+ private final int HEADER_BYTES = 4 * COMPRESSED_INDEX;
+
+ /** Create a PFor compressor/decompressor. */
+ public PFor() {
+ }
+
+ /** Buffer to hold the compressed PFor data.
+ * When the buffer is not large enough, ArrayIndexOutOfBoundExceptions will occur
+ * during compression/decompression.
+ * Without a valid buffer, compress() will only determine the number of bytes needed in the buffer,
+ * see compress().
+ * Without a valid buffer, decompress() will throw a NullPointerException.
+ * For optimal speed when the IntBuffer is a view on a ByteBuffer,
+ * the IntBuffer should have a byte offset of a multiple of 4 bytes, possibly 0.
+ * An IntBuffer is used here because an int has 32 bits, which is always larger
+ * than the number of frame bits, and 32 bits can efficiently accessed in the buffer
+ * on all current processors.
+ */
+ public void setBuffer(IntBuffer intBuffer) {
+ this.intBuffer = intBuffer;
+ }
+
+ /** Compress a given int[] into the buffer.
+ *
+ * When setBuffer() was not done, no actual compression is done.
+ * Regardless of the use of setBuffer(), bufferByteSize() will return
+ * a valid value after calling compress().
+ *
+ * When a buffer is available, the following is done.
+ * A header is stored into the buffer, encoding a.o. numFrameBits and inputSize.
+ * All ints < 2**numFrameBits are stored sequentially in compressed form
+ * in the buffer.
+ * All other ints are stored in the buffer as exceptions after the compressed sequential ints,
+ * using 1, 2 or 4 bytes per exception, starting at the first byte after the compressed
+ * sequential ints.
+ *
+ * The index of the first exception is encoded in the header in the buffer,
+ * all later exceptions have the offset to the next exception as their value,
+ * the last one offset to just after the available input size.
+ * After the first exception, when the next exception index does not fit in
+ * numFrameBits bits, an exception after 2**numFrameBits inputs is forced and inserted.
+ *
+ * Exception values are stored in the order of the exceptions.
+ * The number of bytes used for an exception is also encoded in the header.
+ * This depends on the maximum exception value and does not vary between the exceptions.
+ */
+ public void compress(int[] input, int inputOffset, int inputSize, int numFrameBits) {
+ assert numFrameBits >= 1;
+ assert numFrameBits <= 32;
+ this.numFrameBits = numFrameBits;
+ this.inputSize = inputSize;
+ numExceptions = 0;
+ int maxException = -1;
+ firstExceptionIndex = -1;
+ int lastExceptionIndex = -1;
+ int i;
+ int[] exceptionValues = new int[inputSize];
+ int maxNonExceptionMask = (int) ((1L << numFrameBits) - 1);
+ int maxChain = 254; // maximum value of firstExceptionIndex in header
+ // CHECKME: maxChain 1 off because of initial value of lastExceptionIndex and force exception test below?
+ for (i = 0; i < inputSize; i++) {
+ int v = input[i + inputOffset];
+ // FIXME: split this loop to avoid if statement in loop.
+ // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this.
+ if ( (((v & maxNonExceptionMask) == v) // no value exception
+ && (i < (lastExceptionIndex + maxChain)))) { // no forced exception
+ encodeCompressedValue(i, v); // normal encoding
+ } else { // exception
+ exceptionValues[numExceptions] = v;
+ numExceptions++;
+ if (firstExceptionIndex == -1) {
+ firstExceptionIndex = i;
+ assert firstExceptionIndex <= 254; // maximum value of firstExceptionIndex in header
+ maxException = v;
+ maxChain = 1 << ((30 < numFrameBits) ? 30 : numFrameBits); // fewer bits available for exception chain value.
+ } else if (v > maxException) {
+ maxException = v;
+ }
+ // encode the previous exception pointer
+ if (lastExceptionIndex >= 0) {
+ encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1);
+ }
+ lastExceptionIndex = i;
+ }
+ }
+ if (lastExceptionIndex >= 0) {
+ encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain.
+ }
+ int bitsInArray = numFrameBits * input.length;
+ int bytesInArray = (bitsInArray + 7) / 8;
+ if (maxException < (1 << 8)) { // exceptions as byte
+ exceptionCode = 0;
+ } else if (maxException < (1 << 16)) { // exceptions as 2 bytes
+ exceptionCode = 1;
+ } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes
+ exceptionCode = 2;
+ }
+ encodeHeader(inputSize, firstExceptionIndex);
+ encodeExceptionValues(exceptionValues);
+ }
+
+ /** As compress(), but use the result of getNumFrameBits() as the number of frame bits. */
+ public void compress(int[] input, int inputOffset, int inputSize) {
+ compress(input, inputOffset, inputSize,
+ getNumFrameBits(input, inputOffset, inputSize));
+ }
+
+ private int compressedArrayByteSize() { // override to constant for fixed numFrameBits and inputSize.
+ int compressedArrayBits = inputSize * numFrameBits;
+ return (compressedArrayBits + 7) / 8;
+ }
+
+ /** Return the number bytes used for a single exception */
+ private int exceptionByteSize() {
+ assert exceptionCode >= 0;
+ assert exceptionCode <= 2;
+ return exceptionCode == 0 ? 1
+ : exceptionCode == 1 ? 2
+ : 4;
+ }
+
+ /** Return the number of bytes used in intBuffer.
+ * Only valid after compress() or decompress().
+ */
+ public int bufferByteSize() {
+ return HEADER_BYTES
+ + compressedArrayByteSize()
+ + exceptionByteSize() * numExceptions; // numExceptions only valid after compress() or decompress()
+ }
+
+ private void encodeExceptionValues(int[] exceptionValues) {
+ int excByteOffset = compressedArrayByteSize();
+ for (int i = 0; i < numExceptions; i++) {
+ encodeCompressedValueBase(excByteOffset++, exceptionValues[i] & 255, 8); // will use one int in buffer.
+ if (exceptionCode >= 1) { // double byte
+ encodeCompressedValueBase(excByteOffset++, (exceptionValues[i] >>> 8) & 255, 8); // id.
+ if (exceptionCode == 2) { // quadruple byte
+ encodeCompressedValueBase(excByteOffset++, (exceptionValues[i] >>> 16) & 255, 8); // id.
+ encodeCompressedValueBase(excByteOffset++, (exceptionValues[i] >>> 24) & 255, 8); // id.
+ }
+ }
+ }
+ }
+
+ private void encodeCompressedValue(int compressedPos, int value) {
+ encodeCompressedValueBase(compressedPos, value, numFrameBits);
+ }
+
+ /** Encode a value into the compressed array of numFrameBit bit values by setting the corresponding bits.
+ * Since numFrameBits is always smaller than the number of bits in an int,
+ * at most two ints in the buffer will be affected.
+ * Has no effect when intBuffer == null.
+ */
+ private void encodeCompressedValueBase(int compressedPos, int value, int numBits) {
+ assert numBits >= 0;
+ assert numBits <= 32;
+ final int mask = (int) ((1L << numBits) - 1);
+ if (intBuffer == null) {
+ return;
+ }
+ int compressedBitPos = numBits * compressedPos;
+ int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5);
+ int firstBitPosition = compressedBitPos & 31;
+ assert (intBuffer.get(intIndex) & (mask << firstBitPosition)) == 0; // no bits set yet.
+ intBuffer.put(intIndex, intBuffer.get(intIndex) | (value << firstBitPosition));
+ if ((firstBitPosition + numBits) > 32) { // value does not fit in first int
+ intIndex++;
+ assert (intBuffer.get(intIndex) & (mask >>> (32 - firstBitPosition))) == 0; // no bits set yet.
+ intBuffer.put(intIndex, intBuffer.get(intIndex) | (value >>> (32 - firstBitPosition)));
+ }
+ }
+
+ private int decodeExceptionByte(int compressedBytePos) {
+ final int numBits = 8;
+ final int byteMask = (int) ((1 << numBits) - 1);
+ int compressedBitPos = numBits * compressedBytePos;
+ int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5);
+ int firstBitPosition = compressedBitPos & 31;
+ return (intBuffer.get(intIndex) >>> firstBitPosition) & byteMask;
+ }
+
+ /** The 4 byte header (32 bits) contains:
+ * - 2 bits for the exception size: 0b00: byte, 0b01: short, 0b10: int, 0b11: long (unused).
+ * - 3 bits for the compression method: 0b001 for PFor
+ * - 5 bits for (numFrameBits-1)
+ * - 8 bits for the input size: (128 == 2 ** 7, so 7 bits for input size - 1 would fit)
+ * - 8 bits for the index of the first exception + 1
+ * - 6 bits unused (could be used for the total number of exceptions, but this is redundant).
+ */
+ private void encodeHeader(int inputSize, int firstExceptionIndex) {
+ assert exceptionCode >= 0;
+ assert exceptionCode <= 2; // 3 for long, but unused for now.
+ assert numFrameBits >= 1;
+ assert numFrameBits <= 32;
+ assert inputSize >= 1;
+ assert inputSize <= 128;
+ assert firstExceptionIndex >= -1;
+ assert firstExceptionIndex < inputSize;
+ if (intBuffer != null) {
+ intBuffer.put(HEADER_INDEX,
+ ((firstExceptionIndex+1) << 24)
+ | (inputSize << 16)
+ | ((numFrameBits-1) << 8)
+ | (PFOR_COMPRESSION << 5) | exceptionCode);
+ }
+ }
+
+
+ private void decodeHeader() {
+ if (inputSize != -1) {
+ return;
+ }
+ int header = intBuffer.get(HEADER_INDEX);
+ firstExceptionIndex = ((header >>> 24) & 255) - 1;
+ inputSize = (header >>> 16) & 255;
+ numFrameBits = ((header >>> 8) & 31) + 1;
+ compressionMethod = (header >>> 5) & 7;
+ assert compressionMethod == PFOR_COMPRESSION;
+ exceptionCode = header & 31;
+ assert exceptionCode <= 2;
+ }
+
+ /** Decompress from the buffer into output from a given offset. */
+ public void decompress(int[] output, int outputOffset) {
+ decodeHeader();
+ decodeCompressedInts(output, outputOffset);
+ decodeExceptions(output, outputOffset);
+ }
+
+ /** Return the size of the integer array that is compressed in the buffer. */
+ public int getIntSize() {
+ decodeHeader();
+ return inputSize;
+ }
+
+ /** For performance, this delegates classes with fixed numFrameBits. */
+ private void decodeCompressedInts(int[] output, int outputOffset) {
+ switch (numFrameBits) {
+ case 1:
+ For1Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 2:
+ For2Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 3:
+ For3Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 4:
+ For4Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 5:
+ For5Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 6:
+ For6Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 7:
+ For7Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 8:
+ For8Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 9:
+ For9Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 10:
+ For10Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 11:
+ For11Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 12:
+ For12Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 13:
+ For13Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 14:
+ For14Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 15:
+ For15Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 16:
+ For16Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 17:
+ For17Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 18:
+ For18Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 19:
+ For19Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 20:
+ For20Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 21:
+ For21Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 22:
+ For22Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 23:
+ For23Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 24:
+ For24Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 25:
+ For25Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 26:
+ For26Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 27:
+ For27Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 28:
+ For28Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 29:
+ For29Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 30:
+ For30Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ case 31:
+ For31Decompress.decodeCompressedInts(intBuffer, COMPRESSED_INDEX, inputSize, output, outputOffset);
+ break;
+ default:
+ ForDecompress.decodeAnyFrame(intBuffer, COMPRESSED_INDEX, inputSize, numFrameBits, output, outputOffset);
+ }
+ }
+
+ /** Patch and return index of next exception */
+ static int patch(int[] output, int outputOffset, int excIndex, int excValue) {
+ int nextExceptionIndex = output[excIndex] + excIndex + 1; // chain offset
+ output[excIndex + outputOffset] = excValue; // patch
+ assert nextExceptionIndex > excIndex;
+ return nextExceptionIndex;
+ }
+
+ /** Decode the exception values while going through the exception chain.
+ *
For performance, delegate/subclass this to classes with fixed exceptionCode.
+ *
Also, decoding exceptions is preferably done from an int border instead of
+ * from a random byte directly after the compressed array. This will allow faster
+ * decoding of exceptions, at the cost of at most 3 bytes.
+ *
When ((numFrameBits * inputSize) % 32) == 0, this cost will always be
+ * zero bytes so specialize for these cases.
+ */
+ private void decodeExceptions(int[] output, int outputOffset) {
+ numExceptions = 0;
+ if (firstExceptionIndex == -1) {
+ return;
+ }
+ int excIndex = firstExceptionIndex;
+ int excByteOffset = compressedArrayByteSize(); // constant for given numFrameBits and numFrameBits.
+ int excValue;
+ do {
+ excValue = decodeExceptionByte(excByteOffset++); // will use one int in buffer.
+ if (exceptionCode >= 1) { // double byte
+ excValue |= (decodeExceptionByte(excByteOffset++) << 8);
+ if (exceptionCode == 2) { // quadruple byte
+ excValue |= (decodeExceptionByte(excByteOffset++) << 16);
+ excValue |= (decodeExceptionByte(excByteOffset++) << 24);
+ }
+ }
+ excIndex = patch(output, outputOffset, excIndex, excValue);
+ numExceptions++;
+ } while (excIndex < inputSize);
+ }
+
+ /** Determine the number of frame bits to be used for compression
+ * of a given input array.
+ * This is done by taking a copy of the input, sorting it and using this
+ * to determine the compressed size for each possible numbits in a single pass,
+ * ignoring forced exceptions.
+ * This is basically the method described by Zukowski, but simplified because
+ * the frame of reference is assumed to have 0 as lowerbound.
+ * Finally an estimation of the number of forced exceptions is reduced to
+ * less than 1 in 32 input numbers by increasing the number of frame bits.
+ */
+ public static int getNumFrameBits(int[] input, int inputOffset, int inputSize) {
+ if ((inputOffset + inputSize) > input.length) {
+ throw new IllegalArgumentException( "(inputOffSet " + inputOffset
+ + " + inputSize " + inputSize
+ + ") > input.length " + input.length);
+ }
+ int copy[] = Arrays.copyOfRange(input, inputOffset, inputOffset + inputSize);
+ assert copy.length == inputSize;
+ Arrays.sort(copy);
+ int maxValue = copy[copy.length-1];
+ if (maxValue <= 1) {
+ return 1;
+ }
+ int bytesPerException = (maxValue < (1 << 8)) ? 1 : (maxValue < (1 << 16)) ? 2 : 4;
+ int numFrameBits = 1;
+ int bytesForArray = (copy.length * numFrameBits + 7) / 8;
+ // initially assume all input is an exception.
+ int totalBytes = bytesForArray + copy.length * bytesPerException; // excluding the header.
+ int bestBytes = totalBytes;
+ int bestNumFrameBits = numFrameBits;
+ int bestNumExceptions = copy.length;
+ for (int i = 0; i < copy.length; i++) {
+ totalBytes -= bytesPerException;
+ while (copy[i] >= (1 << numFrameBits)) {
+ if (numFrameBits == 30) { // no point to increase further.
+ return bestNumFrameBits;
+ }
+ ++numFrameBits;
+ while (bytesForArray * 8 < copy.length * numFrameBits) {
+ bytesForArray++;
+ totalBytes++;
+ }
+ }
+ if (totalBytes <= bestBytes) { // <= : prefer fewer exceptions for decompression speed.
+ bestBytes = totalBytes;
+ bestNumFrameBits = numFrameBits;
+ bestNumExceptions = (copy.length - i - 1);
+ }
+ }
+ if (bestNumExceptions == 0) { // no need to bother about forced exceptions.
+ return bestNumFrameBits;
+ }
+ int minNumExceptions = (copy.length >> bestNumFrameBits);
+ int allowedNumExceptions = bestNumExceptions + (copy.length >> 5); // 1 in 32 is allowed to be forced.
+ if (allowedNumExceptions < minNumExceptions) {
+ // Estimation of number of too many forced exceptions positive.
+ bestNumFrameBits++; // reduce forced exceptions and perhaps reduce actual exceptions
+ // minNumExceptions >>= 1; // use this when changing last if to while.
+ }
+ return bestNumFrameBits;
+ }
+}
+
+/* PFor exception decompression. */
+/** Specialisation for single byte exceptions. Not yet used. */
+class P1ForDecompress {
+ static int decodeExceptions(
+ IntBuffer intBuffer,
+ int inputSize,
+ int firstExceptionIndex,
+ int excIntOffset,
+ int[] output,
+ int outputOffset) {
+ int numExceptions = 0;
+ if (firstExceptionIndex != -1) {
+ int excIndex = firstExceptionIndex;
+ do {
+ int excIntValue = intBuffer.get(excIntOffset);
+ excIndex = PFor.patch(output, outputOffset, excIndex, excIntValue & 255);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIndex = PFor.patch(output, outputOffset, excIndex, (excIntValue >> 8) & 255);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIndex = PFor.patch(output, outputOffset, excIndex, (excIntValue >> 16) & 255);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIndex = PFor.patch(output, outputOffset, excIndex, (excIntValue >> 24) & 255);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIntOffset++;
+ } while (true);
+ }
+ return numExceptions;
+ }
+}
+
+/** Specialisation of exception decoding for two byte exceptions. Not yet used. */
+class P2ForDecompress {
+ static int decodeExceptions(
+ IntBuffer intBuffer,
+ int inputSize,
+ int firstExceptionIndex,
+ int excIntOffset,
+ int[] output,
+ int outputOffset) {
+ int numExceptions = 0;
+ if (firstExceptionIndex != -1) {
+ int excIndex = firstExceptionIndex; // in compressed array, and output.
+ final int twoByteMask = (1<<16) - 1;
+ do {
+ int excIntValue = intBuffer.get(excIntOffset);
+ excIndex = PFor.patch(output, outputOffset, excIndex, excIntValue & twoByteMask);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIndex = PFor.patch(output, outputOffset, excIndex, (excIntValue >> 16) & twoByteMask);
+ numExceptions++;
+ if (excIndex >= inputSize) break;
+ excIntOffset++;
+ } while (true);
+ }
+ return numExceptions;
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/pfor/ForDecompress.java
===================================================================
--- src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0)
+++ src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0)
@@ -0,0 +1,65 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.IntBuffer;
+
+/** PFor frame decompression for any number of frame bits. */
+class ForDecompress {
+
+ static void decodeAnyFrame(
+ IntBuffer intBuffer, int bufIndex, int inputSize, int numFrameBits,
+ int[] output, int outputOffset) {
+
+ assert numFrameBits > 0 : numFrameBits;
+ assert numFrameBits <= 32 : numFrameBits;
+ final int mask = (int) ((1L<>> bitPos) & mask;
+ if (--inputSize == 0) return;
+ bitPos += numFrameBits;
+ }
+
+ int intValue2 = intBuffer.get(++bufIndex);
+ output[++outputOffset] = ( (bitPos == 32)
+ ? intValue2
+ : ((intValue1 >>> bitPos) | (intValue2 << (32 - bitPos)))
+ ) & mask;
+
+ if (--inputSize == 0) return;
+
+ intValue1 = intValue2;
+ bitPos += numFrameBits - 32;
+ } while (true);
+ }
+}
Index: src/java/org/apache/lucene/util/pfor/gendecompress.py
===================================================================
--- src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0)
+++ src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0)
@@ -0,0 +1,107 @@
+"""
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""
+
+"""
+Generate source code for java classes for FOR decompression.
+"""
+
+def bitsExpr(i, numFrameBits):
+ framePos = i * numFrameBits
+ intValNum = (framePos / 32)
+ bitPos = framePos % 32
+ bitsInInt = "intValue" + str(intValNum)
+ needBrackets = 0
+ if bitPos > 0:
+ bitsInInt += " >>> " + str(bitPos)
+ needBrackets = 1
+ if bitPos + numFrameBits > 32:
+ if needBrackets:
+ bitsInInt = "(" + bitsInInt + ")"
+ bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")"
+ needBrackets = 1
+ if bitPos + numFrameBits != 32:
+ if needBrackets:
+ bitsInInt = "(" + bitsInInt + ")"
+ bitsInInt += " & mask"
+ return bitsInInt
+
+
+def genDecompressClass(numFrameBits):
+ className = "For" + str(numFrameBits) + "Decompress"
+ fileName = className + ".java"
+ imports = "import java.nio.IntBuffer;\n"
+ f = open(fileName, 'w')
+ w = f.write
+ try:
+ w("package org.apache.lucene.util.pfor;\n")
+ w("""/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */""")
+ w("\n/* This program is generated, do not modify. See gendecompress.py */\n\n")
+ w("import java.nio.IntBuffer;\n")
+ w("class " + className + " extends ForDecompress {\n")
+ w(" static final int numFrameBits = " + str(numFrameBits) + ";\n")
+ w(" static final int mask = (int) ((1L<= 32) {\n")
+ for i in range(numFrameBits): # declare int vars and init from buffer
+ w(" int intValue" + str(i) + " = intBuffer.get(bufIndex")
+ if i > 0:
+ w(" + " + str(i))
+ w(");\n")
+ for i in range(32): # set output from int vars
+ w(" output[" + str(i) + " + outputOffset] = " + bitsExpr(i, numFrameBits) + ";\n")
+ w(""" inputSize -= 32;
+ outputOffset += 32;
+ bufIndex += numFrameBits;
+ }
+
+ if (inputSize > 0)
+ decodeAnyFrame(intBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset);
+ }
+}
+""")
+ finally: f.close()
+
+
+
+def genDecompressClasses():
+ numFrameBits = 1
+ while numFrameBits <= 31:
+ genDecompressClass(numFrameBits)
+ numFrameBits += 1
+
+
+
+if __name__ == "__main__":
+ genDecompressClasses()
\ No newline at end of file
Index: src/test/org/apache/lucene/util/pfor/TestPFor.java
===================================================================
--- src/test/org/apache/lucene/util/pfor/TestPFor.java (revision 0)
+++ src/test/org/apache/lucene/util/pfor/TestPFor.java (revision 0)
@@ -0,0 +1,572 @@
+package org.apache.lucene.util.pfor;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* When using the Sun 1.6 jvm, the performance tests below (using doPerfTestNoExceptions)
+ * should be run with the -server argument to the forked jvm that is used for the
+ * junit tests by adding this line just before the 1st batchtest line
+ * in common-build.xml:
+
+ * Using this -server may be slow for other tests, in particular for shorter tests.
+ */
+
+import junit.framework.TestCase;
+
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+import java.util.Arrays;
+
+public class TestPFor extends TestCase {
+ private void showByte(int b, StringBuffer buf) {
+ for (int i = 7; i >= 0; i--) {
+ buf.append((b >>> i) & 1);
+ }
+ }
+
+ private void showBytes(byte[] array) {
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < array.length; i++) {
+ showByte(array[i] & 255, buf);
+ if (((i+1) % 4) != 0) {
+ buf.append(' ');
+ } else {
+ System.out.println(buf);
+ buf.setLength(0);
+ }
+ }
+ }
+
+ /** Run compression without buffer, return the buffer byte size needed for compression. */
+ private int doNoBufferRun(int[] input, int offset, int numFrameBits) {
+ PFor pforNoBufferCompress = new PFor();
+ pforNoBufferCompress.compress(input, offset, input.length - offset, numFrameBits);
+ return pforNoBufferCompress.bufferByteSize();
+ }
+
+ /** Create an IntBuffer, compress the given input into this buffer, and return it. */
+ private IntBuffer compressToBuffer(int[] input, int offset, int numFrameBits, int bufferByteSize) {
+ // Allocate an IntBuffer as a view on a ByteBuffer
+ int bufferIntSize = bufferByteSize / 4;
+ if (bufferIntSize * 4 < bufferByteSize) bufferIntSize++;
+ ByteBuffer byteBuffer = ByteBuffer.allocate(4 * bufferIntSize);
+ assert byteBuffer.hasArray();
+ byte[] bufferByteArray = byteBuffer.array();
+ assert bufferByteArray != null;
+ IntBuffer intBuffer = byteBuffer.asIntBuffer(); // no offsets used here.
+
+ // Compress to buffer:
+ PFor pforCompress = new PFor();
+ pforCompress.setBuffer(intBuffer);
+ pforCompress.compress(input, offset, input.length - offset, numFrameBits);
+// assert bufferByteArray.length == 4 * bufferIntSize; // for showBytes() below.
+// showBytes(bufferByteArray);
+ if (bufferByteSize >= 0) {
+ assertEquals("Buffer byte size after compress() to buffer", bufferByteSize, pforCompress.bufferByteSize());
+ }
+ int numInputBytes = (input.length - offset) * 4;
+ if (numInputBytes != 0) {
+// System.out.println("Compressed " + numInputBytes + " bytes into "
+// + pforCompress.bufferByteSize()
+// + ", ratio " + (pforCompress.bufferByteSize()/(float)numInputBytes));
+ }
+ return intBuffer;
+ }
+
+ private void deCompressFromBufferVerify(IntBuffer intBuffer, int[] input, int offset, int bufferByteSize) {
+ // Decompress from the buffer:
+ PFor pforDecompress = new PFor();
+ pforDecompress.setBuffer(intBuffer);
+ assertEquals("Decompressed length before decompress()", input.length - offset, pforDecompress.getIntSize());
+ int[] output = new int[input.length]; // use same offset as input
+ pforDecompress.decompress(output, offset);
+ assertEquals("Buffer byte size after decompress()", bufferByteSize, pforDecompress.bufferByteSize());
+ if (! Arrays.equals(input, output)) {
+ for (int i = 0; i < input.length; i++) {
+ System.out.print("at index " + i + " output " + output[i]);
+ System.out.print((input[i] != output[i]) ? " !=" : " ==");
+ System.out.println(" input " + input[i]);
+ }
+ assertEquals("equal array lengths", input.length, output.length);
+ assertTrue("input == output", Arrays.equals(input, output));
+ }
+ }
+
+ private void doTestOffset(int[] input, int offset, int numFrameBits, int bufferByteSize) {
+System.out.println();
+System.out.println(getName());
+ int actBufferByteSize = doNoBufferRun(input, offset, numFrameBits);
+ assertEquals("Buffer byte size after noBuffer run compress()", bufferByteSize, actBufferByteSize);
+ IntBuffer intBuffer = compressToBuffer(input, offset, numFrameBits, actBufferByteSize);
+ // Decompress and verify against original input.
+ deCompressFromBufferVerify(intBuffer, input, offset, actBufferByteSize);
+ }
+
+ private void doTest(int[] input, int numBits, int bufferByteSize) {
+ int offset = 0;
+ doTestOffset(input, offset, numBits, bufferByteSize);
+ }
+
+ public void test01NoExc() {
+ int[] input = {1}; // no exception
+ int numBits = 1;
+ int bufferByteSize = 5;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test02ExcByte() {
+ int[] input = {2}; // 1 byte exception
+ int numBits = 1;
+ int bufferByteSize = 6;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test03ExcTwoByte() {
+ int[] input = {1<<8}; // 2 byte exception
+ int numBits = 1;
+ int bufferByteSize = 7;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test05ExcThreeByte() {
+ int[] input = {1<<16}; // 4 byte exception
+ int numBits = 1;
+ int bufferByteSize = 9;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test06ExcFourByte() {
+ int[] input = {1<<30}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values.
+ int numBits = 1;
+ int bufferByteSize = 9;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test07_Offset1() {
+ int[] input = {0,1};
+ int offset = 1;
+ int numBits = 1;
+ int bufferByteSize = 5;
+ doTestOffset(input, offset, numBits, bufferByteSize);
+ }
+
+ public void test07_Offset2() {
+ int[] input = new int[10];
+ int offset = 9;
+ input[offset] = 1;
+ int numBits = 1;
+ int bufferByteSize = 5;
+ doTestOffset(input, offset, numBits, bufferByteSize);
+ }
+
+ public void test08ForcedException1() {
+ int[] input = {2,1,1}; // 2 exceptions, 1 byte
+ int numBits = 1;
+ int bufferByteSize = 4 + 1 + 2;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test09ForcedException2() {
+ int[] input = {(1<<24),1,0}; // 2 exceptions, 4 byte
+ int numBits = 1;
+ int bufferByteSize = 4 + 1 + 2 * 4;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test10FirstException() {
+ int[] input = {0,1,2,3,0,1,6,7,8}; // Test for not forcing first exception at index 4 (2nd value 0)
+ int numBits = 2;
+ int bufferByteSize = 4 + (numBits * input.length / 8 + 1) + 3; // 3 exceptions from value 6
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test11Series8Base3() { // This also tests for not forcing first exception
+ int[] input = {0,1,2,3,4,5,6,7,8};
+ int numBits = 3;
+ int bufferByteSize = 4 + (numBits * input.length / 8 + 1) + 1; // 1 exception
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test12Series8Base4() {
+ int[] input = {0,1,2,3,4,5,6,7,8};
+ int numBits = 4;
+ int bufferByteSize = 4 + (numBits * input.length / 8 + 1);
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test13Series8Base5() {
+ int[] input = {0,1,2,3,4,5,6,7,8};
+ int numBits = 5;
+ int bufferByteSize = 4 + (numBits * input.length / 8 + 1);
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ private void numFrameBitsTest(int[] input, int expectedNumFrameBits) {
+ System.out.println();
+ System.out.println(getName());
+ assertEquals("numFrameBits", expectedNumFrameBits, PFor.getNumFrameBits(input, 0, input.length));
+ }
+
+ public void test20getNumFrameBits() {
+ int[] input = {2};
+ numFrameBitsTest(input, 2);
+ }
+
+ public void test21getNumFrameBits() {
+ int[] input = {9,8,7,6,5,4,3,2,1,0};
+ numFrameBitsTest(input, 4);
+ }
+
+ public void test22getNumFrameBits() {
+ int[] input = {16000,16001,6,5,4,3,2,1,0};
+ numFrameBitsTest(input, 3);
+ }
+
+ private void noBufferCompressionTest(int[] input) {
+System.out.println();
+System.out.println(getName());
+ // Run compression without buffer:
+ final int offset = 0;
+ PFor pforNoBufferCompress = new PFor();
+ pforNoBufferCompress.compress(input, offset, input.length - offset);
+ int numInputBytes = input.length * 4;
+System.out.println("Compress w/o buffer " + numInputBytes + " bytes into "
+ + pforNoBufferCompress.bufferByteSize()
+ + ", ratio " + (pforNoBufferCompress.bufferByteSize()/(float)numInputBytes));
+ }
+
+ public void test30NoBufferCompression() {
+ int[] input = {0,1,0,1,0,1,0,70000}; // would force exceptions for numFrameBits == 1
+ noBufferCompressionTest(input);
+ }
+
+ public void test31NoBufferCompression() {
+ int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454};
+ noBufferCompressionTest(input);
+ }
+
+ public void test32NoBufferCompression() {
+ int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454,
+ 9,8,7,6,5,4,3,2,1,0,0};
+ noBufferCompressionTest(input);
+ }
+
+ public void test40For1Decompress() {
+ int[] input = {
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0};
+ int numBits = 1;
+ int bufferByteSize = 12;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test41For2Decompress() {
+ int[] input = {
+ 1,0,3,2,2,3,0,1,
+ 1,0,3,2,2,3,0,1,
+ 1,0,3,2,2,3,0,1,
+ 1,0,3,2};
+ int numBits = 2;
+ int bufferByteSize = 11;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test42For3Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 7,5,4,5,6,7,0,1,
+ 1,0,3,6,4,7,5,1,
+ 1,0,4,5,6,7,0,1, // 32 input, 3 ints compressed
+ 1,0,4,5,6,7,0,1,
+ 4,6,3,6,4,7,5,1,
+ 1,0,4,5,6,7}; // 22 more input, 9 bytes compressed
+ int numBits = 3;
+ int bufferByteSize = 4 + 4*3 + 9; // 25
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test43For4Decompress() {
+ int[] input = {
+ 1,0,3,2,5,7,4,6,
+ 8,9,10,2,15,0};
+ int numBits = 4;
+ int bufferByteSize = 11;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test447For17Decompress() {
+ int[] input = {1,1022,1023};
+ int numBits = 17;
+ int bufferByteSize = 4 + ((3 * 17 + 7)/ 8);
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ public void test449For17Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,32768,32767,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,65536,65535}; // 32 input, 17 ints compressed
+ int numBits = 17;
+ int bufferByteSize = 72;
+ doTest(input, numBits, bufferByteSize);
+ }
+
+ private void doPerfTestNoExceptions(int[] input, int numBits) {
+System.out.println();
+System.out.println(getName() + " starting, numFrameBits " + numBits);
+ assert input.length % 32 == 0 : input.length;
+ int bufferByteSize = doNoBufferRun(input, 0, numBits);
+ int sizeNoExceptions = 4 + (input.length * numBits + 7) / 8;
+ assertEquals("Performance test without exceptions, buffer byte size ", sizeNoExceptions, bufferByteSize);
+ IntBuffer intBuffer = compressToBuffer(input, 0, numBits, bufferByteSize);
+ // Verify that decompression is correct:
+ deCompressFromBufferVerify(intBuffer, input, 0, bufferByteSize);
+
+ // Repeat decompressing from the buffer, report on performance.
+ PFor pforDecompress = new PFor();
+ pforDecompress.setBuffer(intBuffer);
+ for (int rep = 0; rep < 3; rep++) {
+ int[] output = new int[input.length]; // use 0 offset
+ long maxTestMillis = 500;
+ long testMillis;
+ int iterations = 0;
+ long startMillis = System.currentTimeMillis();
+ final int decompsPerIter = 1024 * 128;
+ do {
+ for (int i = 0; i < decompsPerIter; i++) {
+ pforDecompress.decompress(output, 0);
+ }
+ iterations++;
+ testMillis = System.currentTimeMillis() - startMillis;
+ } while ((testMillis < maxTestMillis) && (iterations < 1000));
+ long totalDecompressed = (((long) input.length) * decompsPerIter * iterations);
+ System.out.println(getName() + " " + rep
+ + " decompressed " + totalDecompressed
+ + " in " + testMillis + " msecs, "
+ + ((int)(totalDecompressed/((float)testMillis))) + " ints/msec, ("
+ + iterations + " iters).");
+ }
+ }
+
+
+ public void test9PerfFor01Decompress() {
+ int[] input = {
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0,
+ 1,0,1,0,1,0,1,0}; // 32 input, int compressed
+ int numBits = 1;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor02Decompress() {
+ int[] input = {
+ 1,0,3,2,3,2,1,0,
+ 1,0,3,2,3,2,1,0,
+ 1,0,3,2,3,1,0,1,
+ 1,0,3,2,3,2,1,0}; // 32 input, 2 ints compressed
+ int numBits = 2;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor03Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 7,5,4,5,6,7,0,1,
+ 1,0,3,6,4,7,5,1,
+ 1,0,4,5,6,7,0,1}; // 32 input, 3 ints compressed
+ int numBits = 3;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor04Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 9,8,11,14,12,15,13,9,
+ 7,5,4,5,6,7,0,1,
+ 9,8,11,14,12,15,13,9}; // 32 input, 4 ints compressed
+ int numBits = 4;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor05Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 9,8,11,14,12,15,13,9,
+ 23,21,20,21,22,23,16,17,
+ 9,8,11,14,12,15,13,9}; // 32 input, 5 ints compressed
+ int numBits = 5;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor06Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 33,32,35,37,63,62,61,60,
+ 9,8,11,14,12,15,13,9,
+ 23,21,20,21,22,23,16,17}; // 32 input, 6 ints compressed
+ int numBits = 6;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor07Decompress() {
+ int[] input = {
+ 1,0,3,2,7,6,5,4,
+ 33,32,35,37,63,62,61,60,
+ 9,8,11,14,127,126,13,9,
+ 23,21,20,21,22,23,16,17}; // 32 input, 7 ints compressed
+ int numBits = 7;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor08Decompress() {
+ int[] input = {
+ 1,0,3,127,126,13,4,7,
+ 33,32,35,37,63,62,61,60,
+ 9,8,11,14,127,126,13,9,
+ 23,21,20,21,226,255,16,17}; // 32 input, 8 ints compressed
+ int numBits = 8;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor09Decompress() {
+ int[] input = {
+ 1,0,3,127,126,13,4,7,
+ 33,32,35,37,63,62,61,60,
+ 9,8,11,14,511,510,13,9,
+ 23,21,20,21,226,255,16,17}; // 32 input, 9 ints compressed
+ int numBits = 9;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor10Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,4,7,
+ 33,32,35,37,63,62,61,60,
+ 9,8,11,14,511,510,13,9,
+ 23,21,20,21,226,255,16,17}; // 32 input, 10 ints compressed
+ int numBits = 10;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor11Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,4,7,
+ 33,32,35,37,63,2046,2047,60,
+ 9,8,11,14,511,510,13,9,
+ 23,21,20,21,226,255,16,17}; // 32 input, 11 ints compressed
+ int numBits = 11;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor12Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,4,7,
+ 33,32,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,20,21,226,255,16,17}; // 32 input, 12 ints compressed
+ int numBits = 12;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor13Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,4,7,
+ 33,32,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,16,17}; // 32 input, 13 ints compressed
+ int numBits = 13;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor14Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,4,7,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,16,17}; // 32 input, 14 ints compressed
+ int numBits = 14;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor15Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,32766,32767,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,16,17}; // 32 input, 15 ints compressed
+ int numBits = 15;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor16Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,32768,32767,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,65534,65535}; // 32 input, 16 ints compressed
+ int numBits = 16;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor17Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,32768,32767,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,510,13,9,
+ 23,21,8190,8191,226,255,65536,65535}; // 32 input, 17 ints compressed
+ int numBits = 17;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor18Decompress() {
+ int[] input = {
+ 1,1022,1023,127,126,13,32768,32767,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,131071,131072,9,
+ 23,21,8190,8191,226,255,65536,65535}; // 32 input, 18 ints compressed
+ int numBits = 18;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor19Decompress() {
+ int[] input = {
+ 1,1022,1023,127,262144,262143,4,7,
+ 16383,16382,35,37,63,2046,2047,60,
+ 9,4094,4095,14,511,131071,131072,9,
+ 23,21,8190,8191,226,255,16,17}; // 32 input, 19 ints compressed
+ int numBits = 19;
+ doPerfTestNoExceptions(input, numBits);
+ }
+
+ public void test9PerfFor20_31Decompress() {
+ for (int numBits = 20; numBits <= 32; numBits++) {
+ int[] input = {
+ 1,(int)((1L<