Index: src/java/org/apache/lucene/util/pfor/FrameOfRef.java =================================================================== --- src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) @@ -0,0 +1,323 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +import java.util.Arrays; + +import org.apache.lucene.util.BitUtil; + +/** Frame of Reference lossless integer compression/decompression. + * For positive integers, the compression is done by leaving out + * the most significant bits, and storing all numbers with a fixed number of bits + * contiguously in a buffer of bits. This buffer is called the frame, and it + * can store positive numbers in a range from 0 to a constant maximum fitting in + * the number of bits available for a single compressed number. + *

+ * This implementation uses 0 as the lower bound reference for the frame, + * so small positive integers can be most effectively compressed. + *

+ * Optimized code is used for decompression, see class ForDecompress and its subclasses. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * This class does not provide delta coding because the Lucene index + * structures already have that. + *

+ * To be done: + *

+ */ +public class FrameOfRef { + /** Number of frame bits. 2**numFrameBits - 1 is the maximum compressed value. */ + protected int numFrameBits; + + /** Constant header tag to allow other compression methods, use value 0001 for + * Frame of reference. + * CHECKME: Move this to another class defining various decompression methods? + */ + protected int compressionMethod; + private final int FOR_COMPRESSION = 1; /** encode compression method in header */ + + /** IntBuffer for compressed data */ + protected IntBuffer compressedBuffer; + + /** Index of header in int buffer */ + protected final int HEADER_INDEX = 0; + + /** Start index in int buffer of array integers each compressed to numFrameBits. */ + protected final int COMPRESSED_INDEX = HEADER_INDEX + 1; + protected final int HEADER_SIZE = 1; // one integer in IntBuffer + + + /** Uncompressed data */ + protected int[] unCompressedData; + /** Offset into unCompressedData */ + protected int offset; + /** Size of unCompressedData, -1 when not available. */ + protected int unComprSize = -1; + + /** Create a Frame of Reference integer compressor/decompressor. */ + public FrameOfRef() { + } + + /** Integer buffer to hold the compressed data.
+ * Compression and decompression do not affect the current buffer position, + * and the beginning of the compressed data should be or will be at the current + * buffer position.
+ * When the buffer is not large enough, ArrayIndexOutOfBoundExceptions will occur + * during compression/decompression.
+ * Without a buffer for compressed data, compress() will only determine the number + * of integers needed in the buffer, see compress().
+ * Without a valid buffer, decompress() will throw a NullPointerException.
+ * For optimal speed when the IntBuffer is a view on a ByteBuffer, + * the IntBuffer should have a byte offset of a multiple of 4 bytes, possibly 0.
+ * An IntBuffer is used here because 32 bits can efficiently accessed in the buffer + * on all current processors, and a positive int is normally large enough + * for various purposes in a Lucene index. + * + * @param compressedBuffer The buffer to hold the compressed integers. + * + */ + public void setCompressedBuffer(IntBuffer compressedBuffer) { + this.compressedBuffer = compressedBuffer; + } + + + /** Array with offset holding uncompressed data. + * @param unCompressedData The array holding uncompressed integers. + * @param offset offset in unCompressedData. + * @param unComprSize The number of uncompressed integers, should be at least 1. + */ + public void setUnCompressedData(int[] unCompressedData, int offset, int unComprSize) { + assert unCompressedData != null; + assert offset >= 0; + assert unComprSize >= 1; + assert (offset + unComprSize) <= unCompressedData.length; + this.unCompressedData = unCompressedData; + this.offset = offset; + this.unComprSize = unComprSize; + } + + /** Compress the uncompressed data into the buffer using the given number of + * frame bits, storing only this number of least significant bits of the + * uncompressed integers in the compressed buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored as a first integer into the buffer, encoding + * the compression method, the number of frame bits and the number of compressed integers. + * All uncompressed integers are stored sequentially in compressed form + * in the buffer after the header. + * + * @param numFrameBits The number of frame bits. Should be between 1 and 32. + * Note that when this value is 32, no compression occurs. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + encodeHeader(unComprSize); + for (int i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + encodeCompressedValue(i, v); + } + } + + /** As compress(), using the result of frameBitsForCompression() as the number of frame bits. */ + public void compress() { + compress( frameBitsForCompression()); + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + return HEADER_SIZE + (unComprSize * numFrameBits + 31) / 32; + } + + /** Encode an integer value by compressing it into the buffer. + * @param compressedPos The index of the compressed integer in the compressed buffer. + * @param value The non negative value to be stored in compressed form. + * This should fit into the number of frame bits. + */ + protected void encodeCompressedValue(int compressedPos, int value) { + encodeCompressedValueBase(compressedPos, value, numFrameBits); // FIXME: inline private method. + } + + /** Encode a value into the compressed buffer. + * Since numFrameBits is always smaller than the number of bits in an int, + * at most two ints in the buffer will be affected. + *
Has no effect when compressedBuffer == null. + *
This could be specialized for numBits just like decompressFrame(). + */ + private void encodeCompressedValueBase(int compressedPos, int value, int numBits) { + assert numBits >= 1; + assert numBits <= 32; + if (compressedBuffer == null) { + return; + } + final int mask = (int) ((1L << numBits) - 1); + assert ((value & mask) == value) : ("value " + value + ", mask " + mask + ", numBits " + numBits); // lossless compression + final int compressedBitPos = numBits * compressedPos; + final int firstBitPosition = compressedBitPos & 31; + int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5); + setBufferIntBits(intIndex, firstBitPosition, numBits, value); + if ((firstBitPosition + numBits) > 32) { // value does not fit in first int + setBufferIntBits(intIndex+1, 0, (firstBitPosition + numBits - 32), (value >>> (32 - firstBitPosition))); + } + } + + /** Change bits of an integer in the compressed buffer. + *
A more efficient implementation is possible when the compressed + * buffer is known to contain only zero bits, in that case one mask operation can be removed. + * @param intIndex The index of the affected integer in the compressed buffer. + * @param firstBitPosition The position of the least significant bit to be changed. + * @param numBits The number of more significant bits to be changed. + * @param value The new value of the bits to be changed, with the least significant bit at position zero. + */ + protected void setBufferIntBits(int intIndex, int firstBitPosition, int numBits, int value) { + final int mask = (int) ((1L << numBits) - 1); + compressedBuffer.put(intIndex, + (compressedBuffer.get(intIndex) + & ~ (mask << firstBitPosition)) // masking superfluous on clear buffer + | (value << firstBitPosition)); + } + + /** The 4 byte header (32 bits) contains: + *

+ */ + private void encodeHeader(int unComprSize) { + assert numFrameBits >= 1; + assert numFrameBits <= (1 << 5); // 32 + assert unComprSize >= 1; + assert unComprSize <= (1 << 8); // 256 + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((unComprSize-1) << 16) + | ((numFrameBits-1) << 8) + | (FOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(HEADER_INDEX); + unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + compressionMethod = (header >>> 4) & 15; + assert compressionMethod == FOR_COMPRESSION : compressionMethod; + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + decodeHeader(); + decompressFrame(); + } + + /** Return the number of integers available for decompression. + * Do not use before an IntBuffer was passed to setCompressBuffer. + */ + public int decompressedSize() { + decodeHeader(); + return unComprSize; + } + + /** For performance, this delegates to classes with fixed numFrameBits. */ + private void decompressFrame() { + switch (numFrameBits) { + // CHECKME: two other implementations might be faster: + // - array of static methods: Method[numFrameBits].invoke(null, [this]), + // - array of non static decompressors: ForDecompressor[numFrameBits].decompressFrame(this) . + case 1: For1Decompress.decompressFrame(this); break; + case 2: For2Decompress.decompressFrame(this); break; + case 3: For3Decompress.decompressFrame(this); break; + case 4: For4Decompress.decompressFrame(this); break; + case 5: For5Decompress.decompressFrame(this); break; + case 6: For6Decompress.decompressFrame(this); break; + case 7: For7Decompress.decompressFrame(this); break; + case 8: For8Decompress.decompressFrame(this); break; + case 9: For9Decompress.decompressFrame(this); break; + case 10: For10Decompress.decompressFrame(this); break; + case 11: For11Decompress.decompressFrame(this); break; + case 12: For12Decompress.decompressFrame(this); break; + case 13: For13Decompress.decompressFrame(this); break; + case 14: For14Decompress.decompressFrame(this); break; + case 15: For15Decompress.decompressFrame(this); break; + case 16: For16Decompress.decompressFrame(this); break; + case 17: For17Decompress.decompressFrame(this); break; + case 18: For18Decompress.decompressFrame(this); break; + case 19: For19Decompress.decompressFrame(this); break; + case 20: For20Decompress.decompressFrame(this); break; + case 21: For21Decompress.decompressFrame(this); break; + case 22: For22Decompress.decompressFrame(this); break; + case 23: For23Decompress.decompressFrame(this); break; + case 24: For24Decompress.decompressFrame(this); break; + case 25: For25Decompress.decompressFrame(this); break; + case 26: For26Decompress.decompressFrame(this); break; + case 27: For27Decompress.decompressFrame(this); break; + case 28: For28Decompress.decompressFrame(this); break; + case 29: For29Decompress.decompressFrame(this); break; + case 30: For30Decompress.decompressFrame(this); break; + case 31: For31Decompress.decompressFrame(this); break; + case 32: For32Decompress.decompressFrame(this); break; + default: + throw new IllegalStateException("Unknown number of frame bits " + numFrameBits); + } + } + + public int getNumFrameBits() { + return numFrameBits; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * @return The number of bits needed to encode the maximum positive uncompressed value. + * Negative uncompressed values have no influence on the result. + */ + public int frameBitsForCompression() { + int maxNonNegVal = 0; + for (int i = offset; i < (offset + unComprSize); i++) { + if (unCompressedData[i] > maxNonNegVal) { + maxNonNegVal = unCompressedData[i]; + } + } + return BitUtil.logNextHigherPowerOfTwo(maxNonNegVal) + 1; + } +} Index: src/java/org/apache/lucene/util/pfor/PFor.java =================================================================== --- src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) @@ -0,0 +1,415 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +import java.util.Arrays; + +/** Patched Frame of Reference PFOR compression/decompression. + *

+ * As defined in:
+ * Super-Scalar RAM-CPU Cache Compression
+ * Marcin Zukowski, Sándor Héman, Niels Nes, Peter Boncz, 2006.
+ * with extensions from:
+ * Performance of Compressed Inverted List Caching in Search Engines
+ * Jiangong Zhang, Xiaohui Long, Torsten Suel, 2008.
+ *

+ * This class does not provide delta coding because the lucene index + * structures already have that. + *

+ * The implementation uses 0 as lower bound for the frame, + * so small positive integers will be most effectively compressed. + *

+ * Some optimized code is used for decompression, + * see class ForDecompress and its subclasses. + *
Good decompression performance will depend on the performance + * of java.nio.IntBuffer indexed get() methods. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * The start point of first exception is at its natural boundary: + * 2 byte exceptions at even byte position, 4 byte at quadruple. + *

+ * To be done: + *

+ */ +public class PFor extends FrameOfRef { + /** Index on input and in compressed frame of first exception, -1 when no exceptions */ + private int firstExceptionIndex; + + /** CHECKME: Move this to another class defining various decompression methods? */ + private final int PFOR_COMPRESSION = 2; /** to encode compression method in header */ + + /** How to encode PFor exceptions: 0: byte, 1: short, 2:int, unused: 3: long */ + private int exceptionCode = -1; + + /** Total number of exception values */ + private int numExceptions; + + /** Create a PFor compressor/decompressor. */ + public PFor() { + } + + /** Compress the decompressed data into the buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored into the buffer, encoding a.o. numFrameBits and unComprSize. + * All ints < 2**numFrameBits are stored sequentially in compressed form + * in the buffer. + * All other ints are stored in the buffer as exceptions after the compressed sequential ints, + * using 1, 2 or 4 bytes per exception, starting at the first byte after the compressed + * sequential ints. + *
+ * The index of the first exception is encoded in the header in the buffer, + * all later exceptions have the offset to the next exception as their value, + * the last one offset to just after the available input size. + * After the first exception, when the next exception index does not fit in + * numFrameBits bits, an exception after 2**numFrameBits inputs is forced and inserted. + *
+ * Exception values are stored in the order of the exceptions. + * The number of bytes used for an exception is also encoded in the header. + * This depends on the maximum exception value and does not vary between the exceptions. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + numExceptions = 0; + int maxException = -1; + firstExceptionIndex = -1; + int lastExceptionIndex = -1; + int i; + int[] exceptionValues = new int[unComprSize]; + int maxNonExceptionMask = (int) ((1L << numFrameBits) - 1); + int maxChain = 254; // maximum value of firstExceptionIndex in header + // CHECKME: maxChain 1 off because of initial value of lastExceptionIndex and force exception test below? + for (i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + // FIXME: split this loop to avoid if statement in loop. + // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this. + if ( (((v & maxNonExceptionMask) == v) // no value exception + && (i < (lastExceptionIndex + maxChain)))) { // no forced exception + encodeCompressedValue(i, v); // normal encoding + } else { // exception + exceptionValues[numExceptions] = v; + numExceptions++; + if (firstExceptionIndex == -1) { + firstExceptionIndex = i; + assert firstExceptionIndex <= 254; // maximum value of firstExceptionIndex in header + maxException = v; + maxChain = 1 << ((30 < numFrameBits) ? 30 : numFrameBits); // fewer bits available for exception chain value. + } else if (v > maxException) { + maxException = v; + } + // encode the previous exception pointer + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); + } + lastExceptionIndex = i; + } + } + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain. + } + int bitsInArray = numFrameBits * unCompressedData.length; + int bytesInArray = (bitsInArray + 7) / 8; + if (maxException < (1 << 8)) { // exceptions as byte + exceptionCode = 0; + } else if (maxException < (1 << 16)) { // exceptions as 2 bytes + exceptionCode = 1; + } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes + exceptionCode = 2; + } + encodeHeader(unComprSize, firstExceptionIndex); + encodeExceptionValues(exceptionValues); + } + + /** Return the number bytes used for a single exception */ + private int exceptionByteSize() { + assert exceptionCode >= 0; + assert exceptionCode <= 2; + return exceptionCode == 0 ? 1 + : exceptionCode == 1 ? 2 + : 4; + } + + /** Return the number of exceptions. + * Only valid after compress() or decompress(). + */ + public int getNumExceptions() { + return numExceptions; + } + + private int compressedArrayByteSize() { + int compressedArrayBits = unComprSize * numFrameBits; + return (compressedArrayBits + 7) / 8; + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + // numExceptions only valid after compress() or decompress() + return HEADER_SIZE + + ((compressedArrayByteSize() + + exceptionByteSize() * numExceptions + + 3) >> 2); // round up to next multiple of 4 and divide by 4 + } + + private void encodeExceptionValues(int[] exceptionValues) { + if ((compressedBuffer == null) || (numExceptions == 0)) { + return; + } + int excByteOffset = compressedArrayByteSize(); + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + int i = 0; + do { + int intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); // round down here. + setBufferIntBits(intIndex, ((excByteOffset & 3) * 8), 8, exceptionValues[i]); + excByteOffset++; + } while (++i < numExceptions); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + int intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { // encode first 2 byte exception in high 2 bytes of same int as last frame bits. + setBufferIntBits(intIndex, 16, 16, exceptionValues[i]); + intIndex++; + i++; + } + for (; i < (numExceptions-1); i += 2) { + compressedBuffer.put(intIndex++, (exceptionValues[i+1] << 16) | exceptionValues[i]); + } + if (i < numExceptions) { + compressedBuffer.put(intIndex, exceptionValues[i]); // also clear the high 16 bits + } + } + break; + + case 2: { // 4 byte exceptions + int excIntOffSet = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + int i = 0; + do { + compressedBuffer.put(excIntOffSet + i, exceptionValues[i]); + } while(++i < numExceptions); + } + break; + } + } + + /** Decode the exception values while going through the exception chain. + *
For performance, delegate/subclass this to classes with fixed exceptionCode. + *
Also, decoding exceptions is preferably done from an int border instead of + * from a random byte directly after the compressed array. This will allow faster + * decoding of exceptions, at the cost of at most 3 bytes. + *
When ((numFrameBits * unComprSize) % 32) == 0, this cost will always be + * zero bytes so specialize for these cases. + */ + private void patchExceptions() { + numExceptions = 0; + if (firstExceptionIndex == -1) { + return; + } + int excIndex = firstExceptionIndex; + int excByteOffset = compressedArrayByteSize(); + int excValue; + int intIndex; + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + do { + intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); + int firstBitPosition = (excByteOffset & 3) << 3; + excValue = (compressedBuffer.get(intIndex) >>> firstBitPosition) & ((1 << 8) - 1); + excIndex = patch(excIndex, excValue); + excByteOffset++; + } while (excIndex < unComprSize); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { + // decode first 2 byte exception from high 2 bytes of same int as last frame bits. + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + while (excIndex < unComprSize) { + excValue = compressedBuffer.get(intIndex) & ((1<<16)-1); + excIndex = patch(excIndex, excValue); + if (excIndex >= unComprSize) { + break; + } + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + } + break; + + case 2: // 4 byte exceptions + intIndex = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + do { + excValue = compressedBuffer.get(intIndex++); + excIndex = patch(excIndex, excValue); + } while (excIndex < unComprSize); + break; + } + } + + /** The 4 byte header (32 bits) contains: + * + * - 4 bits for the compression method: 0b0001 for PFor + * - 4 bits unused + * + * - 5 bits for (numFrameBits-1) + * - 2 bits for the exception code: 0b00: byte, 0b01: short, 0b10: int, 0b11: long (unused). + * - 1 bit unused + * + * - 8 bits for uncompressed input size - 1, + * + * - 8 bits for the index of the first exception + 1, (0 when no exceptions) + */ + private void encodeHeader(int unComprSize, int firstExceptionIndex) { + assert exceptionCode >= 0; + assert exceptionCode <= 2; // 3 for long, but unused for now. + assert numFrameBits >= 1; + assert numFrameBits <= 32; + assert unComprSize >= 1; + assert unComprSize <= 128; + assert firstExceptionIndex >= -1; + assert firstExceptionIndex < unComprSize; + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((firstExceptionIndex+1) << 24) + | ((unComprSize-1) << 16) + | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) + | (PFOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(HEADER_INDEX); + firstExceptionIndex = ((header >>> 24) & 255) - 1; + unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + assert numFrameBits > 0: numFrameBits; + assert numFrameBits <= 32: numFrameBits; + compressionMethod = (header >>> 4) & 15; + assert compressionMethod == PFOR_COMPRESSION : compressionMethod; + exceptionCode = (header >>> 13) & 3; + assert exceptionCode <= 2; + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + super.decompress(); + patchExceptions(); + } + + /** Patch and return index of next exception */ + private int patch(int excIndex, int excValue) { + int nextExceptionIndex = unCompressedData[excIndex] + excIndex + 1; // chain offset + unCompressedData[excIndex + offset] = excValue; // patch + assert nextExceptionIndex > excIndex; + numExceptions++; + return nextExceptionIndex; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * This is done by taking a copy of the input, sorting it and using this + * to determine the compressed size for each possible numbits in a single pass, + * ignoring forced exceptions. + * Finally an estimation of the number of forced exceptions is reduced to + * less than 1 in 32 input numbers by increasing the number of frame bits. + * This implementation works by determining the total number of bytes needed for + * the compressed data, but does take into account alignment of exceptions + * at 2 or 4 byte boundaries. + */ + public int frameBitsForCompression() { + if ((offset + unComprSize) > unCompressedData.length) { + throw new IllegalArgumentException( "(offset " + offset + + " + unComprSize " + unComprSize + + ") > unCompressedData.length " + unCompressedData.length); + } + int copy[] = Arrays.copyOfRange(unCompressedData, offset, offset + unComprSize); + assert copy.length == unComprSize; + Arrays.sort(copy); + int maxValue = copy[copy.length-1]; + if (maxValue <= 1) { + return 1; + } + int bytesPerException = (maxValue < (1 << 8)) ? 1 : (maxValue < (1 << 16)) ? 2 : 4; + int frameBits = 1; + int bytesForFrame = (copy.length * frameBits + 7) / 8; + // initially assume all input is an exception. + int totalBytes = bytesForFrame + copy.length * bytesPerException; // excluding the header. + int bestBytes = totalBytes; + int bestFrameBits = frameBits; + int bestExceptions = copy.length; + for (int i = 0; i < copy.length; i++) { + // determine frameBits so that copy[i] is no more exception + while (copy[i] >= (1 << frameBits)) { + if (frameBits == 30) { // no point to increase further. + return bestFrameBits; + } + ++frameBits; + // increase bytesForFrame and totalBytes to correspond to frameBits + int newBytesForFrame = (copy.length * frameBits + 7) / 8; + totalBytes += newBytesForFrame - bytesForFrame; + bytesForFrame = newBytesForFrame; + } + totalBytes -= bytesPerException; // no more need to store copy[i] as exception + if (totalBytes <= bestBytes) { // <= : prefer fewer exceptions at higher number of frame bits. + bestBytes = totalBytes; + bestFrameBits = frameBits; + bestExceptions = (copy.length - i - 1); + } + } + if (bestExceptions > 0) { // check for forced exceptions. + // This ignores the position of the first exception, for which enough bits are available in the header. + int allowedNumExceptions = bestExceptions + (copy.length >> 5); // 1 in 32 is allowed to be forced. + // (copy.length >> bestFrameBits): Minimum exception chain size including forced ones, + // ignoring the position of the first exception. + while (allowedNumExceptions < (copy.length >> bestFrameBits)) { // Too many forced? + bestFrameBits++; // Reduce forced exceptions and perhaps reduce actual exceptions + // Dilemma: decompression speed reduces with increasing number of frame bits, + // so it may be better to increase no more than once or twice here. + } + } + return bestFrameBits; + } +} Index: src/java/org/apache/lucene/util/pfor/ForDecompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; + +/** PFor frame decompression for any number of frame bits. */ +class ForDecompress { + + static void decodeAnyFrame( + IntBuffer intBuffer, int bufIndex, int inputSize, int numFrameBits, + int[] output, int outputOffset) { + + assert numFrameBits > 0 : numFrameBits; + assert numFrameBits <= 32 : numFrameBits; + final int mask = (int) ((1L<>> bitPos) & mask; + if (--inputSize == 0) return; + bitPos += numFrameBits; + } + + int intValue2 = intBuffer.get(++bufIndex); + output[++outputOffset] = ( (bitPos == 32) + ? intValue2 + : ((intValue1 >>> bitPos) | (intValue2 << (32 - bitPos))) + ) & mask; + + if (--inputSize == 0) return; + + intValue1 = intValue2; + bitPos += numFrameBits - 32; + } while (true); + } +} Index: src/java/org/apache/lucene/util/pfor/For32Decompress.java =================================================================== --- src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) +++ src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) @@ -0,0 +1,27 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +class For32Decompress extends ForDecompress { + static void decompressFrame(FrameOfRef frameOfRef) { + int oldBufPos = frameOfRef.compressedBuffer.position(); + frameOfRef.compressedBuffer.position(frameOfRef.COMPRESSED_INDEX); + frameOfRef.compressedBuffer.get(frameOfRef.unCompressedData, frameOfRef.offset, frameOfRef.unComprSize); + frameOfRef.compressedBuffer.position(oldBufPos); + } +} Index: src/java/org/apache/lucene/util/pfor/gendecompress.py =================================================================== --- src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) +++ src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) @@ -0,0 +1,110 @@ +""" + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +""" +Generate source code for java classes for FOR decompression. +""" + +def bitsExpr(i, numFrameBits): + framePos = i * numFrameBits + intValNum = (framePos / 32) + bitPos = framePos % 32 + bitsInInt = "intValue" + str(intValNum) + needBrackets = 0 + if bitPos > 0: + bitsInInt += " >>> " + str(bitPos) + needBrackets = 1 + if bitPos + numFrameBits > 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")" + needBrackets = 1 + if bitPos + numFrameBits != 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + bitsInInt += " & mask" + return bitsInInt + + +def genDecompressClass(numFrameBits): + className = "For" + str(numFrameBits) + "Decompress" + fileName = className + ".java" + imports = "import java.nio.IntBuffer;\n" + f = open(fileName, 'w') + w = f.write + try: + w("package org.apache.lucene.util.pfor;\n") + w("""/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */""") + w("\n/* This program is generated, do not modify. See gendecompress.py */\n\n") + w("import java.nio.IntBuffer;\n") + w("class " + className + " extends ForDecompress {\n") + w(" static final int numFrameBits = " + str(numFrameBits) + ";\n") + w(" static final int mask = (int) ((1L<= 32) {\n") + for i in range(numFrameBits): # declare int vars and init from buffer + w(" int intValue" + str(i) + " = compressedBuffer.get(bufIndex") + if i > 0: + w(" + " + str(i)) + w(");\n") + for i in range(32): # set output from int vars + w(" output[" + str(i) + " + outputOffset] = " + bitsExpr(i, numFrameBits) + ";\n") + w(""" inputSize -= 32; + outputOffset += 32; + bufIndex += numFrameBits; + } + + if (inputSize > 0) + decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + } +} +""") + finally: f.close() + + + +def genDecompressClasses(): + numFrameBits = 1 + while numFrameBits <= 31: # 32 special case, not generated. + genDecompressClass(numFrameBits) + numFrameBits += 1 + + + +if __name__ == "__main__": + genDecompressClasses() Index: src/java/org/apache/lucene/util/pfor/package.html =================================================================== --- src/java/org/apache/lucene/util/pfor/package.html (revision 0) +++ src/java/org/apache/lucene/util/pfor/package.html (revision 0) @@ -0,0 +1,25 @@ + + + + + + + +Classes dealing with (patched) frame of reference compression and decompression. + + Index: src/java/org/apache/lucene/util/BitUtil.java =================================================================== --- src/java/org/apache/lucene/util/BitUtil.java (revision 727026) +++ src/java/org/apache/lucene/util/BitUtil.java (working copy) @@ -801,7 +801,7 @@ } /** returns the next highest power of two, or the current value if it's already a power of two or zero*/ - public static long nextHighestPowerOfTwo(long v) { + public static long nextHighestPowerOfTwo(long v) { v--; v |= v >> 1; v |= v >> 2; @@ -813,4 +813,26 @@ return v; } + /** Returns the smallest non negative p such that a given value < (2**(p+1)) + * This differs from (63 - java.lang.Long.numberOfLeadingZeros(v)) + * for non positive given values. + */ + public static int logNextHigherPowerOfTwo(long v) { + long vinput = v; // only for assertions below. + int p = 0; + while (v >= (1 << 8)) { + v >>= 8; + p += 8; + } + while (v >= (1 << 1)) { + v >>= 1; + p++; + } + assert (p <= 62) : p; + assert (p == 62) || (vinput < (1L << (p + 1))) : "p " + p + ", vinput " + vinput; + assert (p == 0) || (vinput >= (1L << p)) : "p " + p + ", vinput " + vinput; + assert (vinput <= 0) || (p == (63 - java.lang.Long.numberOfLeadingZeros(vinput))) : "p " + p + ", vinput " + vinput; + return p; + } + } Index: src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java =================================================================== --- src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java (revision 0) +++ src/test/org/apache/lucene/util/pfor/TestFrameOfRef.java (revision 0) @@ -0,0 +1,543 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* When using the Sun 1.6 jvm, the performance tests below (using doDecompPerfTest... methods) + * should be run with the -server argument to the forked jvm that is used for the + * junit tests by adding this line just before the 1st batchtest line + * in common-build.xml: + + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.util.Arrays; +import org.apache.lucene.util.LuceneTestCase; + + +public class TestFrameOfRef extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private void showByte(int b, StringBuffer buf) { // for debugging + for (int i = 7; i >= 0; i--) { + buf.append((b >>> i) & 1); + } + } + + private void showBytes(byte[] array) { // for debugging + StringBuffer buf = new StringBuffer(); + for (int i = 0; i < array.length; i++) { + showByte(array[i] & 255, buf); + if (((i+1) % 4) != 0) { + buf.append(' '); + } else { + System.out.println(buf); + buf.setLength(0); + } + } + } + + /** Run compression without buffer, return the IntBuffer size needed for compression. */ + static int doNoBufferRun(int[] input, int offset, int numFrameBits, FrameOfRef frameOfRef) { + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(numFrameBits); + return frameOfRef.compressedSize(); + } + + FrameOfRef forNoBufferCompress = new FrameOfRef(); + + /** Create an IntBuffer, compress the given input into this buffer, and return it. */ + static IntBuffer compressToBuffer( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Allocate an IntBuffer as a view on a ByteBuffer + ByteBuffer byteBuffer = ByteBuffer.allocate(4 * compressBufferSize); + assert byteBuffer.hasArray(); + byte[] bufferByteArray = byteBuffer.array(); + assert bufferByteArray != null; + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + // Compress to buffer: + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.setCompressedBuffer(compressBuffer); + frameOfRef.compress(numFrameBits); +// assert bufferByteArray.length == 4 * compressBufferSize; // for showBytes() below. +// showBytes(bufferByteArray); + if (compressBufferSize >= 0) { + //assertEquals("IntBuffer size after compress() to buffer", compressBufferSize, pforCompress.compressedSize()); + } + return compressBuffer; + } + + static void deCompressFromBufferVerify( + IntBuffer compressBuffer, + int[] input, + int offset, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Decompress from the buffer: + frameOfRef.setCompressedBuffer(compressBuffer); + assertEquals("Decompressed length before decompress()", input.length - offset, frameOfRef.decompressedSize()); + int[] output = new int[input.length]; // use same offset as input + frameOfRef.setUnCompressedData(output, offset, output.length - offset); + frameOfRef.decompress(); + assertEquals("IntBuffer size after decompress()", compressBufferSize, frameOfRef.compressedSize()); + if (! Arrays.equals(input, output)) { + for (int i = 0; i < input.length; i++) { + System.out.print("at index " + i + " output " + output[i]); + System.out.print((input[i] != output[i]) ? " !=" : " =="); + System.out.println(" input " + input[i]); + } + assertEquals("equal array lengths", input.length, output.length); + assertTrue("input == output", Arrays.equals(input, output)); + } + } + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { +System.out.println(); +System.out.println(testName); + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(input, offset, numFrameBits, frameOfRef); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer compressBuffer = compressToBuffer(input, offset, numFrameBits, actIntBufferSize, frameOfRef2); + // Decompress and verify against original input. + FrameOfRef frameOfRef3 = new FrameOfRef(); + deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, frameOfRef3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test01NoExc() { + int[] input = {1}; // no exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test07_Offset1() { + int[] input = {0,1}; + int offset = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test07_Offset2() { + int[] input = new int[10]; + int offset = 9; + input[offset] = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test12Series8Base4() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test13Series8Base5() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 5; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { +System.out.println(); +System.out.println(getName()); + FrameOfRef frameOfRef = new FrameOfRef(); + frameOfRef.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, frameOfRef.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + static void noBufferCompressionTest(int[] input, FrameOfRef frameOfRef, String testName) { +System.out.println(); +System.out.println(testName); + // Run compression without buffer: + final int offset = 0; + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(); +System.out.println("Compress w/o buffer " + input.length + " ints into " + + frameOfRef.compressedSize() + + ", ratio " + (frameOfRef.compressedSize()/(float)input.length)); + } + + public void test30NoBufferCompression() { + int[] input = {0,1,0,1,0,1,0,70000}; // would force exceptions for numFrameBits == 1 + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test31NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test32NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454, + 9,8,7,6,5,4,3,2,1,0,0}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test40For1Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0}; + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test41For2Decompress() { + int[] input = { + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2}; + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test42For3Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1, // 32 input, 3 ints compressed + 1,0,4,5,6,7,0,1, + 4,6,3,6,4,7,5,1, + 1,0,4,5,6,7}; // 22 more input, 9 bytes compressed + int numBits = 3; + int compressBufferSize = 7; + doTest(input, numBits, compressBufferSize); + } + + public void test43For4Decompress() { + int[] input = { + 1,0,3,2,5,7,4,6, + 8,9,10,2,15,0}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test447For17Decompress() { + int[] input = {1,1022,1023}; + int numBits = 17; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test449For17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + int compressBufferSize = 18; + doTest(input, numBits, compressBufferSize); + } + + /** Repeat decompressing from a given buffer, report on performance. */ + static void doDecompPerfTestUsing( + IntBuffer comprDataBuffer, + int decomprSize, + FrameOfRef frameOfRef, + String testName) { + frameOfRef.setCompressedBuffer(comprDataBuffer); + for (int rep = 0; rep < 3; rep++) { + long maxTestMillis = 300; // 1000 + long testMillis; + int iterations = 0; + long startMillis = System.currentTimeMillis(); + final int decompsPerIter = 1024 * 128; + int[] output = new int[decomprSize]; // use 0 offset + frameOfRef.setUnCompressedData(output, 0, decomprSize); // FIXME: use decomprSize from compressBuffer, or even from pforDecompress + do { + for (int i = 0; i < decompsPerIter; i++) { + frameOfRef.decompress(); + } + iterations++; + testMillis = System.currentTimeMillis() - startMillis; + } while ((testMillis < maxTestMillis) && (iterations < 1000)); + long totalDecompressed = (((long) decomprSize) * decompsPerIter * iterations); + System.out.println(testName + " " + rep + + " bits " + frameOfRef.getNumFrameBits() + + " decompressed " + totalDecompressed + + " in " + testMillis + " msecs, " + + ((int)(totalDecompressed/(testMillis * 1000f))) + " kints/msec, (" + + iterations + " iters)."); + } + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprSize, frameOfRef, getName()); + } + + private void doCompDecompPerfTest(int[] decomprData, int numBits) { + int offset = 0; + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(decomprData, offset, numBits, frameOfRef); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer comprDataBuffer = compressToBuffer(decomprData, offset, numBits, actIntBufferSize, frameOfRef2); + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef3 = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprData.length, frameOfRef3, getName()); + } + + public void test91PerfFor01Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0}; + int numBits = 1; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor02Decompress() { + int[] input = { + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,1,0,1, + 1,0,3,2,3,2,1,0}; + int numBits = 2; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor03Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1}; + int numBits = 3; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor04Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 7,5,4,5,6,7,0,1, + 9,8,11,14,12,15,13,9}; + int numBits = 4; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor05Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17, + 9,8,11,14,12,15,13,9}; + int numBits = 5; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor06Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 6; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor07Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 7; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor08Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 8; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor09Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 9; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor10Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 10; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor11Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 11; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor12Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 12; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor13Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 13; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor14Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 14; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor15Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32766,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 15; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor16Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65534,65535}; + int numBits = 16; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor18Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 18; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor19Decompress() { + int[] input = { + 1,1022,1023,127,262144,262143,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 19; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L< + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.IntBuffer; +import org.apache.lucene.util.LuceneTestCase; + +/** + * To be added: performance tests for decoding exceptions. + */ +public class TestPFor extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { +System.out.println(); +System.out.println(testName); + PFor pFor = new PFor(); + int actIntBufferSize = TestFrameOfRef.doNoBufferRun(input, offset, numFrameBits, pFor); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + PFor pFor2 = new PFor(); + IntBuffer compressBuffer = TestFrameOfRef.compressToBuffer(input, offset, numFrameBits, actIntBufferSize, pFor2); + // Decompress and verify against original input. + PFor pFor3 = new PFor(); + TestFrameOfRef.deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, pFor3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test02ExcByte1() { + int[] input = {2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte2() { + int[] input = {1,2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte3() { + int[] input = {1,(1<<7)}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte4() { + int[] input = {1,(1<<7),0}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte5() { + int[] input = {1,(1<<7),0,65}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte1() { + int[] input = {1<<8}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte2() { + int[] input = {1<<8, 1}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte3() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 3; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte4() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte5() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte6() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test05ExcThreeByte() { + int[] input = {1<<16}; // 4 byte exception + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte1() { + int[] input = {1<<30}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte2() { + int[] input = {1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte3() { + int[] input = {1,1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test08ForcedException1() { + int[] input = {2,1,1}; // 2 exceptions, 1 byte + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test09ForcedException2() { + int[] input = {(1<<24),1,0}; // 2 exceptions, 4 byte + int numBits = 1; + int compressBufferSize = 4; + doTest(input, numBits, compressBufferSize); + } + + public void test10FirstException() { + int[] input = {0,1,2,3,0,1,6,7,8}; // Test for not forcing first exception at index 4 (2nd value 0) + int numBits = 2; + int compressBufferSize = 3; // 3 exceptions from value 6 + doTest(input, numBits, compressBufferSize); + } + + public void test11Series8Base3() { // This also tests for not forcing first exception + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 3; + int compressBufferSize = 3; // 1 exception + doTest(input, numBits, compressBufferSize); + } + + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { +System.out.println(); +System.out.println(getName()); + PFor pfor = new PFor(); + pfor.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, pfor.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + public void test22frameBitsForCompression() { + int[] input = {16000,16001,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 3); + } + + private void doDecompPerfTest(int[] decomprData, int numBitsNoExceptions) { + doDecompPerfTestNoExceptions(decomprData, numBitsNoExceptions); + for (int nb = numBitsNoExceptions-1; nb >= numBitsNoExceptions-3; nb--) { + if (nb <= 0) + break; + doDecompPerfTestExceptions(decomprData, nb); + } + } + + private void checkExceptions(int inputLength, int numBits, int compressBufferSize, boolean exceptionsPresent) { + int sizeNoExceptions = 1 + (inputLength * numBits + 31) / 32; + if (exceptionsPresent) { + assertTrue(("Performance test with exceptions, IntBuffer " + sizeNoExceptions + " < " + compressBufferSize), + sizeNoExceptions < compressBufferSize); + System.out.println("Nr of exception quadbytes: " + (compressBufferSize - sizeNoExceptions)); + } else { + assertEquals("Performance test without exceptions, IntBuffer size", sizeNoExceptions, compressBufferSize); + } + } + + private void doDecompPerfTestNoExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, false); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + private void doDecompPerfTestExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, true); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + TestFrameOfRef.doDecompPerfTestUsing(comprDataBuffer, decomprSize, new PFor(), getName()); + } + + public void tst92PerfPFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L< \n"); + System.out.println("Eg: java org.apache.lucene.util.TestPFor2 /lucene/index _l.prx _l.prx.pfor\n"); + System.exit(1); + } + + String vIntFileNameIn = args[1]; + String pForFileNameOut = args[2]; + + // Convert vInt encoding --> pfor + if (!dir.fileExists(pForFileNameOut)) { + System.out.println("\nencode " + vIntFileNameIn + " to " + pForFileNameOut + "..."); + convertVIntToPFor(dir, vIntFileNameIn, pForFileNameOut); + } + + System.out.println("\ndecompress using pfor:"); + long bestPFor = 0; + for(int round=0;round<5;round++) { + long speed = readPFor(dir, pForFileNameOut); + if (speed > bestPFor) + bestPFor = speed; + } + + System.out.println("\ndecompress using readVInt:"); + long bestVInt = 0; + for(int round=0;round<5;round++) { + long speed = readVInts(dir, vIntFileNameIn); + if (speed > bestVInt) + bestVInt = speed; + } + + NumberFormat nf = NumberFormat.getInstance(); + if (bestVInt > bestPFor) + System.out.println("\nPFor is " + nf.format((bestVInt-bestPFor)*100.0/bestVInt) + "% slower"); + else + System.out.println("\nPFor is " + nf.format((bestPFor-bestVInt)*100.0/bestVInt) + "% faster"); + + dir.close(); + } + + /** Returns ints/sec speed */ + public static long readVInts(Directory dir, String vIntFileNameIn) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + in.readVInt(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + in.close(); + System.out.println((t1-t0) + " msec to read " + count + " ints (" + (count/(t1-t0)) + " ints/msec)"); + + return count/(t1-t0); + } + + /** Returns ints/sec speed */ + public static long readPFor(Directory dir, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(pForFileNameOut); + + PFor pforDecompress = new PFor(); + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + pforDecompress.setCompressedBuffer(compressBuffer); + final int[] temp = new int[BLOCK_SIZE]; + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + int numByte = in.readInt(); + in.readBytes(bufferByteArray, 0, numByte); + pforDecompress.setUnCompressedData(temp, 0, 0); + pforDecompress.decompress(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + System.out.println((t1-t0) + " msec to decode " + (BLOCK_SIZE*count) + " ints (" + (BLOCK_SIZE*count/(t1-t0)) + " ints/msec)"); + in.close(); + + return (BLOCK_SIZE*count)/(t1-t0); + } + + public static void convertVIntToPFor(Directory dir, String vIntFileNameIn, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + IndexOutput out = dir.createOutput(pForFileNameOut); + + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + PFor pforCompress = new PFor(); + pforCompress.setCompressedBuffer(compressBuffer); + + // Get ints + int count = 0; + int upto = 0; + int[] temp = new int[BLOCK_SIZE]; + + final Random r = new Random(); + final int[] counts = new int[32]; + + while(true) { + try { + temp[upto++] = in.readVInt(); + } catch (IOException ioe) { + break; + } + if (upto == BLOCK_SIZE) { + pforCompress.setUnCompressedData(temp, 0, BLOCK_SIZE); + final int numFrameBits = pforCompress.frameBitsForCompression(); + counts[numFrameBits]++; + pforCompress.compress(); + final int numByte = pforCompress.compressedSize() * 4; + out.writeInt(numByte); + out.writeBytes(bufferByteArray, 0, numByte); + upto = 0; + count++; + } + } + in.close(); + out.close(); + System.out.println((BLOCK_SIZE*count) + " ints; " + dir.fileLength(pForFileNameOut) + " bytes compressed vs orig size " + dir.fileLength(vIntFileNameIn)); + + /* + NumberFormat nf = NumberFormat.getInstance(); + for(int i=1;i<31;i++) + System.out.println(i + " bits: " + counts[i] + " [" + nf.format(100.0*counts[i]/count) + " %]"); + */ + } +} \ No newline at end of file Index: src/test/org/apache/lucene/util/TestBitUtil.java =================================================================== --- src/test/org/apache/lucene/util/TestBitUtil.java (revision 0) +++ src/test/org/apache/lucene/util/TestBitUtil.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +/** + * Tests for class BitUtil. + * Currently only BitUtil.logNextHigherPowerOfTwo() is tested. + * (Most of the BitUtil functionality is tested indirectly via TestBitVector.) + */ +public class TestBitUtil extends LuceneTestCase +{ + private void tstlnp(long v, long p) { + assertEquals("value " + v, p, BitUtil.logNextHigherPowerOfTwo(v)); + } + + public void testLnp01() { + tstlnp(0, 0); + for (int p = 1; p <= 62; p++) { + tstlnp((1L<