Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java (working copy) @@ -50,11 +50,10 @@ @Override protected void flushBlock() throws IOException { - // make sure output is always aligned to int - assert (out.getFilePointer() & 3) == 0; compressor.setUnCompressedData(buffer, 0, buffer.length); final int numFrameBits = compressor.frameBitsForCompression(); - compressor.compress(); + final int header = compressor.compress(); + out.writeByte((byte) header); final int numBytes = compressor.compressedSize() * 4; assert numBytes <= 1024; out.writeBytes(output, numBytes); Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java (working copy) @@ -47,7 +47,7 @@ } public void skipBlock() throws IOException { - int numBytes = in.readInt(); // nocommit: should PFOR use vint header? + int numBytes = ((in.readByte() & 0xff) + 1) << 2; in.seek(in.getFilePointer() + numBytes); // seek past block } } Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java (working copy) @@ -44,14 +44,12 @@ @Override protected void flushBlock() throws IOException { - // make sure output is always aligned to int - assert (out.getFilePointer() & 3) == 0; compressor.setUnCompressedData(buffer, 0, buffer.length); final int numFrameBits = compressor.frameBitsForCompression(); compressor.compress(); final int numBytes = compressor.compressedSize() * 4; assert numBytes <= 1024; - out.writeInt(numBytes); + out.writeByte((byte) (compressor.compressedSize()-1)); out.writeBytes(output, numBytes); } } Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java (working copy) @@ -49,8 +49,7 @@ // nocommit: abstraction, and can this be simplified?! public void skipBlock() throws IOException { - int header = in.readInt(); // should FOR/PFOR use vint header? - final int numFrameBits = ((header >>> 8) & 31) + 1; + int numFrameBits = in.readByte() & 0xff; final int numBytes = numFrameBits << 4; in.seek(in.getFilePointer() + numBytes); // seek past block } Index: lucene/src/java/org/apache/lucene/util/pfor/ForCompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/ForCompress.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/util/pfor/ForCompress.java (working copy) @@ -91,20 +91,21 @@ * @param numFrameBits The number of frame bits. Should be between 1 and 32. * Note that when this value is 32, no compression occurs. */ - public void compress(int numFrameBits) { + public int compress(int numFrameBits) { assert numFrameBits >= 1; assert numFrameBits <= 32; this.numFrameBits = numFrameBits; - encodeHeader(unComprSize); + int header = encodeHeader(unComprSize); for (int i = 0; i < unComprSize; i++) { int v = unCompressedData[i + offset]; - encodeCompressedValue(i, v); + encodeCompressedValue(0, i, v); } + return header; } /** As compress(), using the result of frameBitsForCompression() as the number of frame bits. */ - public void compress() { - compress( frameBitsForCompression()); + public int compress() { + return compress( frameBitsForCompression()); } /** Determine the number of frame bits to be used for compression. @@ -138,17 +139,12 @@ *
  • 8 bits unused. * */ - private void encodeHeader(int unComprSize) { + private int encodeHeader(int unComprSize) { assert numFrameBits >= 1; assert numFrameBits <= (1 << 5); // 32 assert unComprSize >= 1; assert unComprSize <= (1 << 8); // 256 - if (compressedBuffer != null) { - compressedBuffer.put(ForConstants.HEADER_INDEX, - ((unComprSize-1) << 16) - | ((numFrameBits-1) << 8) - | (ForConstants.FOR_COMPRESSION << 4)); - } + return numFrameBits; } /** Encode an integer value by compressing it into the buffer. @@ -156,8 +152,8 @@ * @param value The non negative value to be stored in compressed form. * This should fit into the number of frame bits. */ - protected void encodeCompressedValue(int compressedPos, int value) { - encodeCompressedValueBase(compressedPos, value, numFrameBits); // FIXME: inline private method. + protected void encodeCompressedValue(int headerLen, int compressedPos, int value) { + encodeCompressedValueBase(headerLen, compressedPos, value, numFrameBits); // FIXME: inline private method. } /** Encode a value into the compressed buffer. @@ -166,7 +162,7 @@ *
    Has no effect when compressedBuffer == null. *
    This could be specialized for numBits just like decompressFrame(). */ - private void encodeCompressedValueBase(int compressedPos, int value, int numBits) { + private void encodeCompressedValueBase(int headerLen, int compressedPos, int value, int numBits) { assert numBits >= 1; assert numBits <= 32; if (compressedBuffer == null) { @@ -176,7 +172,7 @@ assert ((value & mask) == value) : ("value " + value + ", mask " + mask + ", numBits " + numBits); // lossless compression final int compressedBitPos = numBits * compressedPos; final int firstBitPosition = compressedBitPos & 31; - int intIndex = ForConstants.COMPRESSED_INDEX + (compressedBitPos >> 5); + int intIndex = headerLen + (compressedBitPos >> 5); setBufferIntBits(intIndex, firstBitPosition, numBits, value); if ((firstBitPosition + numBits) > 32) { // value does not fit in first int setBufferIntBits(intIndex+1, 0, (firstBitPosition + numBits - 32), (value >>> (32 - firstBitPosition))); @@ -203,6 +199,6 @@ * Only valid after compress() or decompress(). */ public int compressedSize() { - return ForConstants.HEADER_SIZE + (unComprSize * numFrameBits + 31) / 32; + return (unComprSize * numFrameBits + 31) / 32; } } Index: lucene/src/java/org/apache/lucene/util/pfor/PForCompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/PForCompress.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/util/pfor/PForCompress.java (working copy) @@ -94,7 +94,7 @@ * This depends on the maximum exception value and does not vary between the exceptions. */ @Override - public void compress(int numFrameBits) { + public int compress(int numFrameBits) { assert numFrameBits >= 1; assert numFrameBits <= 32; this.numFrameBits = numFrameBits; @@ -113,7 +113,7 @@ // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this. if ( (((v & maxNonExceptionMask) == v) // no value exception && (i < (lastExceptionIndex + maxChain)))) { // no forced exception - encodeCompressedValue(i, v); // normal encoding + encodeCompressedValue(ForConstants.HEADER_SIZE, i, v); // normal encoding } else { // exception exceptionValues[numExceptions] = v; numExceptions++; @@ -127,13 +127,13 @@ } // encode the previous exception pointer if (lastExceptionIndex >= 0) { - encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); + encodeCompressedValue(ForConstants.HEADER_SIZE, lastExceptionIndex, i - lastExceptionIndex - 1); } lastExceptionIndex = i; } } if (lastExceptionIndex >= 0) { - encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain. + encodeCompressedValue(ForConstants.HEADER_SIZE, lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain. } //int bitsInArray = numFrameBits * unCompressedData.length; //int bytesInArray = (bitsInArray + 7) / 8; @@ -144,8 +144,9 @@ } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes exceptionCode = 2; } - encodeHeader(unComprSize, firstExceptionIndex); + int ret = encodeHeader(unComprSize, firstExceptionIndex); encodeExceptionValues(exceptionValues); + return ret; } /** Return the number bytes used for a single exception */ @@ -240,7 +241,7 @@ * * - 8 bits for the index of the first exception + 1, (0 when no exceptions) */ - private void encodeHeader(int unComprSize, int firstExceptionIndex) { + private int encodeHeader(int unComprSize, int firstExceptionIndex) { assert exceptionCode >= 0; assert exceptionCode <= 2; // 3 for long, but unused for now. assert numFrameBits >= 1; @@ -249,13 +250,17 @@ assert unComprSize <= 128; assert firstExceptionIndex >= -1; assert firstExceptionIndex < unComprSize; + + final int header = ((firstExceptionIndex+1) << 24) + | ((unComprSize-1) << 16) + | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) + | (ForConstants.PFOR_COMPRESSION << 4); + if (compressedBuffer != null) { - compressedBuffer.put(ForConstants.HEADER_INDEX, - ((firstExceptionIndex+1) << 24) - | ((unComprSize-1) << 16) - | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) - | (ForConstants.PFOR_COMPRESSION << 4)); + compressedBuffer.put(ForConstants.HEADER_INDEX, header); } + + return header; } /** Determine the number of frame bits to be used for compression. Index: lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java (working copy) @@ -79,8 +79,7 @@ /** Decompress from the buffer into output from a given offset. */ public void decompress() throws IOException { - int header = in.readInt(); - final int numFrameBits = ((header >>> 8) & 31) + 1; + int numFrameBits = in.readByte() & 0xff; in.readBytes(input, 0, numFrameBits << 4); compressedBuffer.rewind(); switch (numFrameBits) { Index: lucene/src/java/org/apache/lucene/util/pfor/PForDecompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/PForDecompress.java (revision 1070978) +++ lucene/src/java/org/apache/lucene/util/pfor/PForDecompress.java (working copy) @@ -151,7 +151,7 @@ /** Decompress from the buffer into output from a given offset. */ public void decompress() throws IOException { - int numBytes = in.readInt(); // nocommit: is it possible to encode # of exception bytes in header? + int numBytes = ((in.readByte() & 0xff) + 1) << 2; // nocommit: is it possible to encode # of exception bytes in header? in.readBytes(input, 0, numBytes); compressedBuffer.rewind(); int header = compressedBuffer.get();