Index: lucene/src/test/org/apache/lucene/util/TestBitUtil.java =================================================================== --- lucene/src/test/org/apache/lucene/util/TestBitUtil.java (revision 0) +++ lucene/src/test/org/apache/lucene/util/TestBitUtil.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +/** + * Tests for class BitUtil. + * Currently only BitUtil.logNextHigherPowerOfTwo() is tested. + * (Most of the BitUtil functionality is tested indirectly via TestBitVector.) + */ +public class TestBitUtil extends LuceneTestCase +{ + private void tstlnp(long v, long p) { + assertEquals("value " + v, p, BitUtil.logNextHigherPowerOfTwo(v)); + } + + public void testLnp01() { + tstlnp(0, 0); + for (int p = 1; p <= 62; p++) { + tstlnp((1L< \n"); + System.out.println("Eg: java org.apache.lucene.util.TestPFor2 /lucene/index _l.prx _l.prx.pfor\n"); + System.exit(1); + } + + String vIntFileNameIn = args[1]; + String pForFileNameOut = args[2]; + + // Convert vInt encoding --> pfor + if (!dir.fileExists(pForFileNameOut)) { + System.out.println("\nencode " + vIntFileNameIn + " to " + pForFileNameOut + "..."); + convertVIntToPFor(dir, vIntFileNameIn, pForFileNameOut); + } + + System.out.println("\ndecompress using pfor:"); + long bestPFor = 0; + for(int round=0;round<5;round++) { + long speed = readPFor(dir, pForFileNameOut); + if (speed > bestPFor) + bestPFor = speed; + } + + System.out.println("\ndecompress using readVInt:"); + long bestVInt = 0; + for(int round=0;round<5;round++) { + long speed = readVInts(dir, vIntFileNameIn); + if (speed > bestVInt) + bestVInt = speed; + } + + NumberFormat nf = NumberFormat.getInstance(); + if (bestVInt > bestPFor) + System.out.println("\nPFor is " + nf.format((bestVInt-bestPFor)*100.0/bestVInt) + "% slower"); + else + System.out.println("\nPFor is " + nf.format((bestPFor-bestVInt)*100.0/bestVInt) + "% faster"); + + dir.close(); + } + + /** Returns ints/sec speed */ + public static long readVInts(Directory dir, String vIntFileNameIn) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + in.readVInt(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + in.close(); + System.out.println((t1-t0) + " msec to read " + count + " ints (" + (count/(t1-t0)) + " ints/msec)"); + + return count/(t1-t0); + } + + /** Returns ints/sec speed */ + public static long readPFor(Directory dir, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(pForFileNameOut); + + PFor pforDecompress = new PFor(); + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + pforDecompress.setCompressedBuffer(compressBuffer); + final int[] temp = new int[BLOCK_SIZE]; + final long t0 = System.currentTimeMillis(); + long count = 0; + while(true) { + try { + int numByte = in.readInt(); + in.readBytes(bufferByteArray, 0, numByte); + pforDecompress.setUnCompressedData(temp, 0, 0); + pforDecompress.decompress(); + count++; + } catch (IOException ioe) { + break; + } + } + final long t1 = System.currentTimeMillis(); + System.out.println((t1-t0) + " msec to decode " + (BLOCK_SIZE*count) + " ints (" + (BLOCK_SIZE*count/(t1-t0)) + " ints/msec)"); + in.close(); + + return (BLOCK_SIZE*count)/(t1-t0); + } + + public static void convertVIntToPFor(Directory dir, String vIntFileNameIn, String pForFileNameOut) throws Throwable { + IndexInput in = dir.openInput(vIntFileNameIn); + IndexOutput out = dir.createOutput(pForFileNameOut); + + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + byte[] bufferByteArray = byteBuffer.array(); + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + PFor pforCompress = new PFor(); + pforCompress.setCompressedBuffer(compressBuffer); + + // Get ints + int count = 0; + int upto = 0; + int[] temp = new int[BLOCK_SIZE]; + + final Random r = new Random(); + final int[] counts = new int[32]; + + while(true) { + try { + temp[upto++] = in.readVInt(); + } catch (IOException ioe) { + break; + } + if (upto == BLOCK_SIZE) { + pforCompress.setUnCompressedData(temp, 0, BLOCK_SIZE); + final int numFrameBits = pforCompress.frameBitsForCompression(); + counts[numFrameBits]++; + pforCompress.compress(); + final int numByte = pforCompress.compressedSize() * 4; + out.writeInt(numByte); + out.writeBytes(bufferByteArray, 0, numByte); + upto = 0; + count++; + } + } + in.close(); + out.close(); + System.out.println((BLOCK_SIZE*count) + " ints; " + dir.fileLength(pForFileNameOut) + " bytes compressed vs orig size " + dir.fileLength(vIntFileNameIn)); + + /* + NumberFormat nf = NumberFormat.getInstance(); + for(int i=1;i<31;i++) + System.out.println(i + " bits: " + counts[i] + " [" + nf.format(100.0*counts[i]/count) + " %]"); + */ + } +} \ No newline at end of file Index: lucene/src/test/org/apache/lucene/util/pfor/TestPFor.java =================================================================== --- lucene/src/test/org/apache/lucene/util/pfor/TestPFor.java (revision 0) +++ lucene/src/test/org/apache/lucene/util/pfor/TestPFor.java (revision 0) @@ -0,0 +1,287 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* When using the Sun 1.6 jvm, the performance tests below (using doDecompPerfTest... methods) + * should be run with the -server argument to the forked jvm that is used for the + * junit tests by adding this line just before the 1st batchtest line + * in common-build.xml: + + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.IntBuffer; +import org.apache.lucene.util.LuceneTestCase; + +/** + * To be added: performance tests for decoding exceptions. + */ +public class TestPFor extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { + //System.out.println(); + //System.out.println(testName); + PFor pFor = new PFor(); + int actIntBufferSize = TestFrameOfRef.doNoBufferRun(input, offset, numFrameBits, pFor); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + PFor pFor2 = new PFor(); + IntBuffer compressBuffer = TestFrameOfRef.compressToBuffer(input, offset, numFrameBits, actIntBufferSize, pFor2); + // Decompress and verify against original input. + PFor pFor3 = new PFor(); + TestFrameOfRef.deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, pFor3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test02ExcByte1() { + int[] input = {2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte2() { + int[] input = {1,2}; // 1 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte3() { + int[] input = {1,(1<<7)}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte4() { + int[] input = {1,(1<<7),0}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test02ExcByte5() { + int[] input = {1,(1<<7),0,65}; // 1 byte exception + int numBits = 7; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte1() { + int[] input = {1<<8}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte2() { + int[] input = {1<<8, 1}; // 2 byte exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte3() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 3; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte4() { + int[] input = {1<<8, 1, 2}; // 2 byte exception + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte5() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test03ExcTwoByte6() { + int[] input = {1<<8, 1, 1<<9}; // two 2 byte exceptions + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test05ExcThreeByte() { + int[] input = {1<<16}; // 4 byte exception + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte1() { + int[] input = {1<<30}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte2() { + int[] input = {1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test06ExcFourByte3() { + int[] input = {1,1<<30,0}; // 4 byte exception, (1<<31 is negative, an assertion fails on negative values. + int numBits = 6; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test08ForcedException1() { + int[] input = {2,1,1}; // 2 exceptions, 1 byte + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test09ForcedException2() { + int[] input = {(1<<24),1,0}; // 2 exceptions, 4 byte + int numBits = 1; + int compressBufferSize = 4; + doTest(input, numBits, compressBufferSize); + } + + public void test10FirstException() { + int[] input = {0,1,2,3,0,1,6,7,8}; // Test for not forcing first exception at index 4 (2nd value 0) + int numBits = 2; + int compressBufferSize = 3; // 3 exceptions from value 6 + doTest(input, numBits, compressBufferSize); + } + + public void test11Series8Base3() { // This also tests for not forcing first exception + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 3; + int compressBufferSize = 3; // 1 exception + doTest(input, numBits, compressBufferSize); + } + + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { +System.out.println(); +System.out.println(getName()); + PFor pfor = new PFor(); + pfor.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, pfor.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + public void test22frameBitsForCompression() { + int[] input = {16000,16001,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 3); + } + + private void doDecompPerfTest(int[] decomprData, int numBitsNoExceptions) { + doDecompPerfTestNoExceptions(decomprData, numBitsNoExceptions); + for (int nb = numBitsNoExceptions-1; nb >= numBitsNoExceptions-3; nb--) { + if (nb <= 0) + break; + doDecompPerfTestExceptions(decomprData, nb); + } + } + + private void checkExceptions(int inputLength, int numBits, int compressBufferSize, boolean exceptionsPresent) { + int sizeNoExceptions = 1 + (inputLength * numBits + 31) / 32; + if (exceptionsPresent) { + assertTrue(("Performance test with exceptions, IntBuffer " + sizeNoExceptions + " < " + compressBufferSize), + sizeNoExceptions < compressBufferSize); + System.out.println("Nr of exception quadbytes: " + (compressBufferSize - sizeNoExceptions)); + } else { + assertEquals("Performance test without exceptions, IntBuffer size", sizeNoExceptions, compressBufferSize); + } + } + + private void doDecompPerfTestNoExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, false); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + private void doDecompPerfTestExceptions(int[] decomprData, int numBits) { + assert decomprData.length % 32 == 0 : decomprData.length; + PFor pFor = new PFor(); + int comprDataSize = TestFrameOfRef.doNoBufferRun(decomprData, 0, numBits, pFor); + checkExceptions(decomprData.length, numBits, comprDataSize, true); + IntBuffer comprDataBuffer = TestFrameOfRef.compressToBuffer(decomprData, 0, numBits, comprDataSize, pFor); + // Verify that decompression is correct: + TestFrameOfRef.deCompressFromBufferVerify(comprDataBuffer, decomprData, 0, comprDataSize, pFor); + + if (! doDecompPerfTests) { + return; + } +System.out.println(); +System.out.println(getName() + " starting, numFrameBits " + numBits); + doDecompPerfTest(comprDataBuffer, decomprData.length); + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + TestFrameOfRef.doDecompPerfTestUsing(comprDataBuffer, decomprSize, new PFor(), getName()); + } + + public void tst92PerfPFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L< + * Using this -server may be slow for other tests, in particular for shorter tests. + */ + +import junit.framework.TestCase; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; +import java.util.Arrays; +import org.apache.lucene.util.LuceneTestCase; + + +public class TestFrameOfRef extends LuceneTestCase { + private static boolean doDecompPerfTests = true; + + private void showByte(int b, StringBuffer buf) { // for debugging + for (int i = 7; i >= 0; i--) { + buf.append((b >>> i) & 1); + } + } + + private void showBytes(byte[] array) { // for debugging + StringBuffer buf = new StringBuffer(); + for (int i = 0; i < array.length; i++) { + showByte(array[i] & 255, buf); + if (((i+1) % 4) != 0) { + buf.append(' '); + } else { + System.out.println(buf); + buf.setLength(0); + } + } + } + + /** Run compression without buffer, return the IntBuffer size needed for compression. */ + static int doNoBufferRun(int[] input, int offset, int numFrameBits, FrameOfRef frameOfRef) { + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(numFrameBits); + return frameOfRef.compressedSize(); + } + + FrameOfRef forNoBufferCompress = new FrameOfRef(); + + /** Create an IntBuffer, compress the given input into this buffer, and return it. */ + static IntBuffer compressToBuffer( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Allocate an IntBuffer as a view on a ByteBuffer + ByteBuffer byteBuffer = ByteBuffer.allocate(4 * compressBufferSize); + assert byteBuffer.hasArray(); + byte[] bufferByteArray = byteBuffer.array(); + assert bufferByteArray != null; + IntBuffer compressBuffer = byteBuffer.asIntBuffer(); // no offsets used here. + + // Compress to buffer: + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.setCompressedBuffer(compressBuffer); + frameOfRef.compress(numFrameBits); + // assert bufferByteArray.length == 4 * compressBufferSize; // for showBytes() below. + // showBytes(bufferByteArray); + if (compressBufferSize >= 0) { + //assertEquals("IntBuffer size after compress() to buffer", compressBufferSize, pforCompress.compressedSize()); + } + return compressBuffer; + } + + static void deCompressFromBufferVerify( + IntBuffer compressBuffer, + int[] input, + int offset, + int compressBufferSize, + FrameOfRef frameOfRef) { + // Decompress from the buffer: + frameOfRef.setCompressedBuffer(compressBuffer); + assertEquals("Decompressed length before decompress()", input.length - offset, frameOfRef.decompressedSize()); + int[] output = new int[input.length]; // use same offset as input + frameOfRef.setUnCompressedData(output, offset, output.length - offset); + frameOfRef.decompress(); + assertEquals("IntBuffer size after decompress()", compressBufferSize, frameOfRef.compressedSize()); + if (! Arrays.equals(input, output)) { + for (int i = 0; i < input.length; i++) { + System.out.print("at index " + i + " output " + output[i]); + System.out.print((input[i] != output[i]) ? " !=" : " =="); + System.out.println(" input " + input[i]); + } + assertEquals("equal array lengths", input.length, output.length); + assertTrue("input == output", Arrays.equals(input, output)); + } + } + + private static void doTestOffset( + int[] input, + int offset, + int numFrameBits, + int compressBufferSize, + String testName) { + //System.out.println(); + //System.out.println(testName); + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(input, offset, numFrameBits, frameOfRef); + assertEquals("IntBuffer size after noBuffer run compress()", compressBufferSize, actIntBufferSize); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer compressBuffer = compressToBuffer(input, offset, numFrameBits, actIntBufferSize, frameOfRef2); + // Decompress and verify against original input. + FrameOfRef frameOfRef3 = new FrameOfRef(); + deCompressFromBufferVerify(compressBuffer, input, offset, actIntBufferSize, frameOfRef3); + } + + private void doTest(int[] input, int numBits, int compressBufferSize) { + int offset = 0; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test01NoExc() { + int[] input = {1}; // no exception + int numBits = 1; + int compressBufferSize = 2; + doTest(input, numBits, compressBufferSize); + } + + public void test07_Offset1() { + int[] input = {0,1}; + int offset = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test07_Offset2() { + int[] input = new int[10]; + int offset = 9; + input[offset] = 1; + int numBits = 1; + int compressBufferSize = 2; + doTestOffset(input, offset, numBits, compressBufferSize, getName()); + } + + public void test12Series8Base4() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test13Series8Base5() { + int[] input = {0,1,2,3,4,5,6,7,8}; + int numBits = 5; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + private void frameBitsForCompressionTest(int[] input, int expectedNumFrameBits) { + //System.out.println(); + //System.out.println(getName()); + FrameOfRef frameOfRef = new FrameOfRef(); + frameOfRef.setUnCompressedData(input, 0, input.length); + assertEquals("numFrameBits", expectedNumFrameBits, frameOfRef.frameBitsForCompression()); + } + + public void test20frameBitsForCompression() { + int[] input = {2}; + frameBitsForCompressionTest(input, 2); + } + + public void test21frameBitsForCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0}; + frameBitsForCompressionTest(input, 4); + } + + static void noBufferCompressionTest(int[] input, FrameOfRef frameOfRef, String testName) { + //System.out.println(); + //System.out.println(testName); + // Run compression without buffer: + final int offset = 0; + frameOfRef.setUnCompressedData(input, offset, input.length - offset); + frameOfRef.compress(); + /* + System.out.println("Compress w/o buffer " + input.length + " ints into " + + frameOfRef.compressedSize() + + ", ratio " + + (frameOfRef.compressedSize()/(float)input.length)); + */ + } + + public void test30NoBufferCompression() { + int[] input = {0,1,0,1,0,1,0,70000}; // would force exceptions for numFrameBits == 1 + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test31NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test32NoBufferCompression() { + int[] input = {9,8,7,6,5,4,3,2,1,0,21,22,23,24,22,45,76,223,43,62,454, + 9,8,7,6,5,4,3,2,1,0,0}; + noBufferCompressionTest(input, new FrameOfRef(), getName()); + } + + public void test40For1Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0}; + int numBits = 1; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test41For2Decompress() { + int[] input = { + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2,2,3,0,1, + 1,0,3,2}; + int numBits = 2; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test42For3Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1, // 32 input, 3 ints compressed + 1,0,4,5,6,7,0,1, + 4,6,3,6,4,7,5,1, + 1,0,4,5,6,7}; // 22 more input, 9 bytes compressed + int numBits = 3; + int compressBufferSize = 7; + doTest(input, numBits, compressBufferSize); + } + + public void test43For4Decompress() { + int[] input = { + 1,0,3,2,5,7,4,6, + 8,9,10,2,15,0}; + int numBits = 4; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test447For17Decompress() { + int[] input = {1,1022,1023}; + int numBits = 17; + int compressBufferSize = 3; + doTest(input, numBits, compressBufferSize); + } + + public void test449For17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + int compressBufferSize = 18; + doTest(input, numBits, compressBufferSize); + } + + /** Repeat decompressing from a given buffer, report on performance. */ + static void doDecompPerfTestUsing( + IntBuffer comprDataBuffer, + int decomprSize, + FrameOfRef frameOfRef, + String testName) { + frameOfRef.setCompressedBuffer(comprDataBuffer); + for (int rep = 0; rep < 3; rep++) { + long maxTestMillis = 300; // 1000 + long testMillis; + int iterations = 0; + long startMillis = System.currentTimeMillis(); + final int decompsPerIter = 1024 * 128; + int[] output = new int[decomprSize]; // use 0 offset + frameOfRef.setUnCompressedData(output, 0, decomprSize); // FIXME: use decomprSize from compressBuffer, or even from pforDecompress + do { + for (int i = 0; i < decompsPerIter; i++) { + frameOfRef.decompress(); + } + iterations++; + testMillis = System.currentTimeMillis() - startMillis; + } while ((testMillis < maxTestMillis) && (iterations < 1000)); + long totalDecompressed = (((long) decomprSize) * decompsPerIter * iterations); + /* + System.out.println(testName + " " + rep + + " bits " + frameOfRef.getNumFrameBits() + + " decompressed " + totalDecompressed + + " in " + testMillis + " msecs, " + + ((int)(totalDecompressed/(testMillis * 1000f))) + " kints/msec, (" + + iterations + " iters)."); + */ + } + } + + + private void doDecompPerfTest(IntBuffer comprDataBuffer, int decomprSize) { + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprSize, frameOfRef, getName()); + } + + private void doCompDecompPerfTest(int[] decomprData, int numBits) { + int offset = 0; + FrameOfRef frameOfRef = new FrameOfRef(); + int actIntBufferSize = doNoBufferRun(decomprData, offset, numBits, frameOfRef); + FrameOfRef frameOfRef2 = new FrameOfRef(); + IntBuffer comprDataBuffer = compressToBuffer(decomprData, offset, numBits, actIntBufferSize, frameOfRef2); + if (! doDecompPerfTests) { + return; + } + FrameOfRef frameOfRef3 = new FrameOfRef(); + doDecompPerfTestUsing(comprDataBuffer, decomprData.length, frameOfRef3, getName()); + } + + public void test91PerfFor01Decompress() { + int[] input = { + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0, + 1,0,1,0,1,0,1,0}; + int numBits = 1; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor02Decompress() { + int[] input = { + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,2,1,0, + 1,0,3,2,3,1,0,1, + 1,0,3,2,3,2,1,0}; + int numBits = 2; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor03Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 7,5,4,5,6,7,0,1, + 1,0,3,6,4,7,5,1, + 1,0,4,5,6,7,0,1}; + int numBits = 3; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor04Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 7,5,4,5,6,7,0,1, + 9,8,11,14,12,15,13,9}; + int numBits = 4; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor05Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17, + 9,8,11,14,12,15,13,9}; + int numBits = 5; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor06Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,12,15,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 6; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor07Decompress() { + int[] input = { + 1,0,3,2,7,6,5,4, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,22,23,16,17}; + int numBits = 7; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor08Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,127,126,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 8; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor09Decompress() { + int[] input = { + 1,0,3,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 9; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor10Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,62,61,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 10; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor11Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,8,11,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 11; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor12Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,20,21,226,255,16,17}; + int numBits = 12; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor13Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 33,32,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 13; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor14Decompress() { + int[] input = { + 1,1022,1023,127,126,13,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 14; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor15Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32766,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 15; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor16Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65534,65535}; + int numBits = 16; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor17Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,510,13,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 17; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor18Decompress() { + int[] input = { + 1,1022,1023,127,126,13,32768,32767, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,65536,65535}; + int numBits = 18; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor19Decompress() { + int[] input = { + 1,1022,1023,127,262144,262143,4,7, + 16383,16382,35,37,63,2046,2047,60, + 9,4094,4095,14,511,131071,131072,9, + 23,21,8190,8191,226,255,16,17}; + int numBits = 19; + doCompDecompPerfTest(input, numBits); + } + + public void test91PerfFor20_32Decompress() { + for (int numBits = 20; numBits <= 32; numBits++) { + int[] input = { + 1,(int)((1L<= pointerMax) { + if (!refillBuffer()) { + doc = NO_MORE_DOCS; // set to sentinel value + return false; + } + } + doc += docs[pointer]; + freq = freqs[pointer]; + } + } else { + while (doc < end) { // for docs in window + //System.out.println(" " + doc); + c.collect(doc); // collect score + if (++pointer >= pointerMax) { + if (!refillBuffer()) { + doc = NO_MORE_DOCS; // set to sentinel value + return false; + } + } + doc = docs[pointer]; + freq = freqs[pointer]; + } } return true; } @@ -112,16 +166,23 @@ */ @Override public int nextDoc() throws IOException { + //System.out.println("TS " + term.utf8ToString() + ": nextDoc pointer=" + pointer + " vs max=" + pointerMax); pointer++; if (pointer >= pointerMax) { - refillBuffer(); - if (pointerMax != 0) { - pointer = 0; - } else { + if (!refillBuffer()) { return doc = NO_MORE_DOCS; } } - doc = docs[pointer]; + if (docDeltas) { + if (doc == -1) { + doc = docs[pointer]; + } else { + doc += docs[pointer]; + } + } else { + doc = docs[pointer]; + } + //System.out.println(" " + doc); freq = freqs[pointer]; assert doc != NO_MORE_DOCS; return doc; @@ -149,22 +210,43 @@ */ @Override public int advance(int target) throws IOException { + //System.out.println("TS " + term.utf8ToString() + ": advance t=" + target + " deltas=" + docDeltas + " doc=" + doc + " p=" + pointer + " pMax=" + pointerMax); // first scan in cache for (pointer++; pointer < pointerMax; pointer++) { - if (docs[pointer] >= target) { + if (docDeltas) { + assert docs[pointer] > 0; + doc += docs[pointer]; + } else { + doc = docs[pointer]; + } + if (doc >= target) { + //System.out.println(" in-cache p=" + pointer); + //System.out.println(" found doc=" + doc); freq = freqs[pointer]; - return doc = docs[pointer]; + return doc; } } + //System.out.println(" NOT in-cache"); // not found in readahead cache, seek underlying stream - int newDoc = docsEnum.advance(target); - //System.out.println("ts.advance docsEnum=" + docsEnum); + int newDoc = docsEnum.advance(target, doc); if (newDoc != NO_MORE_DOCS) { doc = newDoc; + //System.out.println(" found doc=" + doc + " p=" + pointer + " pMax=" + pointerMax); freq = docsEnum.freq(); + + // nocommit -- we don't really want to refill here; + // instead we want to "tap in" to the underlying int[] + // buffer if it's already decoded; ie we do not want + // to actually decode the next buffer, if we are on + // the end: + if (docDeltas) { + refillBuffer(); + pointer--; + } } else { doc = NO_MORE_DOCS; + //System.out.println(" found EOF"); } return doc; } Index: lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 981713) +++ lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -107,6 +107,7 @@ // calculate the maximum k edits for this similarity maxEdits = initialMaxDistance(minSimilarity, termLength); + System.out.println("term=" + term + " maxEdits=" + maxEdits); TermsEnum subEnum = getAutomatonEnum(maxEdits, null); setEnum(subEnum != null ? subEnum : Index: lucene/src/java/org/apache/lucene/index/DocsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocsEnum.java (revision 981713) +++ lucene/src/java/org/apache/lucene/index/DocsEnum.java (working copy) @@ -54,6 +54,11 @@ protected BulkReadResult bulkResult; + // nocommit jdoc + public int advance(int target, int curDoc) throws IOException { + return advance(target); + } + protected final void initBulkResult() { if (bulkResult == null) { bulkResult = new BulkReadResult(); @@ -99,4 +104,20 @@ } return count; } + + // nocommit jdocs + public int readDocs() throws IOException { + int count = 0; + final int[] docs = bulkResult.docs.ints; + while(count < docs.length) { + final int doc = nextDoc(); + if (doc != NO_MORE_DOCS) { + docs[count] = doc; + count++; + } else { + break; + } + } + return count; + } } Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexOutput.java (revision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Naive int block API that writes vInts. This is + * expected to give poor performance; it's really only for + * testing the pluggability. One should typically use pfor instead. */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; +import org.apache.lucene.util.pfor.FrameOfRef; +import org.apache.lucene.util.CodecUtil; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +/** Don't use this class!! It naively encodes ints one vInt + * at a time. Use it only for testing. */ +public class FORIndexOutput extends FixedIntBlockIndexOutput { + + public final static String CODEC = "P_FOR_DELTA"; + public final static int VERSION_START = 0; + public final static int VERSION_CURRENT = VERSION_START; + private final FrameOfRef compressor; + private final byte[] output; + + public FORIndexOutput(Directory dir, String fileName, int blockSize) throws IOException { + IndexOutput out = dir.createOutput(fileName); + //CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); + init(out, blockSize); + + compressor = new FrameOfRef(); + // nocommit -- can't hardwire 1024; it's a function of blockSize + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + output = byteBuffer.array(); + compressor.setCompressedBuffer(byteBuffer.asIntBuffer()); + } + + protected void flushBlock(int[] buffer, IndexOutput out) throws IOException { + compressor.setUnCompressedData(buffer, 0, buffer.length); + final int numFrameBits = compressor.frameBitsForCompression(); + compressor.compress(); + final int numBytes = compressor.compressedSize() * 4; + assert numBytes <= 1024; + out.writeInt(numBytes); + out.writeBytes(output, numBytes); + } +} + Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORCodec.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORCodec.java (revision 0) @@ -0,0 +1,146 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.util.Set; +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.SepCodec; +import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl; +import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsDictReader; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader; + +public class FORCodec extends Codec { + + final static String DOC_EXTENSION = "doc"; + final static String SKIP_EXTENSION = "skp"; + final static String FREQ_EXTENSION = "frq"; + final static String PROX_EXTENSION = "prx"; + final static String PAYLOAD_EXTENSION = "pyl"; + + public FORCodec() { + name = "FOR"; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + StandardPostingsWriter postingsWriter = new SepPostingsWriterImpl(state, new FORFactory(128)); + + boolean success = false; + StandardTermsIndexWriter indexWriter; + try { + indexWriter = new SimpleStandardTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator()); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + StandardPostingsReader postingsReader = new SepPostingsReaderImpl(state.dir, + state.segmentInfo, + state.readBufferSize, + new FORFactory(128)); + + StandardTermsIndexReader indexReader; + boolean success = false; + try { + indexReader = new SimpleStandardTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator()); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new StandardTermsDictReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.readBufferSize, + BytesRef.getUTF8SortedAsUnicodeComparator(), + StandardCodec.TERMS_CACHE_SIZE); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, Set files) { + SepPostingsReaderImpl.files(segmentInfo, files); + StandardTermsDictReader.files(dir, segmentInfo, files); + SimpleStandardTermsIndexReader.files(dir, segmentInfo, files); + } + + @Override + public void getExtensions(Set extensions) { + SepCodec.getSepExtensions(extensions); + } +} Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaFactory.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; + +import java.io.IOException; + +public class PForDeltaFactory extends IntStreamFactory { + private final int blockSize; + + /** blockSize is only used when creating the + * IntIndexOutput */ + public PForDeltaFactory(int blockSize) { + this.blockSize = blockSize; + } + + public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + return new PForDeltaIndexInput(dir, fileName, readBufferSize); + } + + public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { + return new PForDeltaIndexOutput(dir, fileName, blockSize); + } +} Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexInput.java (revision 0) @@ -0,0 +1,77 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; +import org.apache.lucene.util.pfor.PFor; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +public class PForDeltaIndexInput extends FixedIntBlockIndexInput { + + public PForDeltaIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException { + IndexInput in = dir.openInput(fileName, readBufferSize); + //CodecUtil.checkHeader(in, PForDeltaIndexOutput.CODEC, PForDeltaIndexOutput.VERSION_START, PForDeltaIndexOutput.VERSION_START); + init(in); + } + + private static class BlockReader implements FixedIntBlockIndexInput.BlockReader { + private final IndexInput in; + private final int[] buffer; + private final PFor decompressor; + private final byte[] input; + private final IntBuffer intInput; + + public BlockReader(IndexInput in, int[] buffer) { + this.in = in; + this.buffer = buffer; + + decompressor = new PFor(); + // nocommit -- can't hardwire 1024; it's a function of blockSize + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + input = byteBuffer.array(); + intInput = byteBuffer.asIntBuffer(); + decompressor.setCompressedBuffer(intInput); + decompressor.setUnCompressedData(buffer, 0, buffer.length); + } + + public void seek(long pos) throws IOException { + // + } + + public void readBlock() throws IOException { + int numBytes = in.readInt(); + //System.out.println("nb=" + numBytes); + // nocommit -- how to avoid this copy? + in.readBytes(input, 0, numBytes); + intInput.rewind(); + decompressor.decompress(); + } + } + + protected BlockReader getBlockReader(IndexInput in, int[] buffer) { + return new BlockReader(in, buffer); + } +} + Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaIndexOutput.java (revision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Naive int block API that writes vInts. This is + * expected to give poor performance; it's really only for + * testing the pluggability. One should typically use pfor instead. */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; +import org.apache.lucene.util.pfor.PFor; +import org.apache.lucene.util.CodecUtil; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +/** Don't use this class!! It naively encodes ints one vInt + * at a time. Use it only for testing. */ +public class PForDeltaIndexOutput extends FixedIntBlockIndexOutput { + + public final static String CODEC = "P_FOR_DELTA"; + public final static int VERSION_START = 0; + public final static int VERSION_CURRENT = VERSION_START; + private final PFor compressor; + private final byte[] output; + + public PForDeltaIndexOutput(Directory dir, String fileName, int blockSize) throws IOException { + IndexOutput out = dir.createOutput(fileName); + //CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); + init(out, blockSize); + + compressor = new PFor(); + // nocommit -- can't hardwire 1024; it's a function of blockSize + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + output = byteBuffer.array(); + compressor.setCompressedBuffer(byteBuffer.asIntBuffer()); + } + + protected void flushBlock(int[] buffer, IndexOutput out) throws IOException { + compressor.setUnCompressedData(buffer, 0, buffer.length); + final int numFrameBits = compressor.frameBitsForCompression(); + compressor.compress(); + final int numBytes = compressor.compressedSize() * 4; + assert numBytes <= 1024; + out.writeInt(numBytes); + out.writeBytes(output, numBytes); + } +} + Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaCodec.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/PForDeltaCodec.java (revision 0) @@ -0,0 +1,146 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.util.Set; +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.SepCodec; +import org.apache.lucene.index.codecs.sep.SepPostingsWriterImpl; +import org.apache.lucene.index.codecs.sep.SepPostingsReaderImpl; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.standard.StandardTermsDictWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsDictReader; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.StandardTermsIndexReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexWriter; +import org.apache.lucene.index.codecs.standard.SimpleStandardTermsIndexReader; + +public class PForDeltaCodec extends Codec { + + final static String DOC_EXTENSION = "doc"; + final static String SKIP_EXTENSION = "skp"; + final static String FREQ_EXTENSION = "frq"; + final static String PROX_EXTENSION = "prx"; + final static String PAYLOAD_EXTENSION = "pyl"; + + public PForDeltaCodec() { + name = "PForDelta"; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + StandardPostingsWriter postingsWriter = new SepPostingsWriterImpl(state, new PForDeltaFactory(128)); + + boolean success = false; + StandardTermsIndexWriter indexWriter; + try { + indexWriter = new SimpleStandardTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new StandardTermsDictWriter(indexWriter, state, postingsWriter, BytesRef.getUTF8SortedAsUnicodeComparator()); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + StandardPostingsReader postingsReader = new SepPostingsReaderImpl(state.dir, + state.segmentInfo, + state.readBufferSize, + new PForDeltaFactory(128)); + + StandardTermsIndexReader indexReader; + boolean success = false; + try { + indexReader = new SimpleStandardTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator()); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new StandardTermsDictReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.readBufferSize, + BytesRef.getUTF8SortedAsUnicodeComparator(), + StandardCodec.TERMS_CACHE_SIZE); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, Set files) { + SepPostingsReaderImpl.files(segmentInfo, files); + StandardTermsDictReader.files(dir, segmentInfo, files); + SimpleStandardTermsIndexReader.files(dir, segmentInfo, files); + } + + @Override + public void getExtensions(Set extensions) { + SepCodec.getSepExtensions(extensions); + } +} Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORFactory.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; + +import java.io.IOException; + +public class FORFactory extends IntStreamFactory { + private final int blockSize; + + /** blockSize is only used when creating the + * IntIndexOutput */ + public FORFactory(int blockSize) { + this.blockSize = blockSize; + } + + public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + return new FORIndexInput(dir, fileName, readBufferSize); + } + + public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { + return new FORIndexOutput(dir, fileName, blockSize); + } +} Index: lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/codecs/pfordelta/FORIndexInput.java (revision 0) @@ -0,0 +1,173 @@ +package org.apache.lucene.index.codecs.pfordelta; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.MMapDirectory.MMapIndexInput; +import org.apache.lucene.store.MMapDirectory.MultiMMapIndexInput; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; +import org.apache.lucene.util.pfor.FrameOfRef; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.IntBuffer; + +public class FORIndexInput extends FixedIntBlockIndexInput { + + public FORIndexInput(Directory dir, String fileName, int readBufferSize) throws IOException { + IndexInput in = dir.openInput(fileName, readBufferSize); + //CodecUtil.checkHeader(in, PForDeltaIndexOutput.CODEC, PForDeltaIndexOutput.VERSION_START, PForDeltaIndexOutput.VERSION_START); + init(in); + } + + private static class BlockReader implements FixedIntBlockIndexInput.BlockReader { + private final IndexInput in; + private final int[] buffer; + private final FrameOfRef decompressor; + private final byte[] input; + private final IntBuffer intInput; + + public BlockReader(IndexInput in, int[] buffer) { + this.in = in; + this.buffer = buffer; + + decompressor = new FrameOfRef(); + // nocommit -- can't hardwire 1024; it's a function of blockSize + ByteBuffer byteBuffer = ByteBuffer.allocate(1024); + input = byteBuffer.array(); + intInput = byteBuffer.asIntBuffer(); + decompressor.setCompressedBuffer(intInput); + decompressor.setUnCompressedData(buffer, 0, buffer.length); + } + + public void seek(long pos) throws IOException { + } + + public void readBlock() throws IOException { + // nocommit -- we don't need this numBytes header -- + // it's a waste. we need something like the zip + // interface -- the decompressor asks for more bytes + // if it needs it + int numBytes = in.readInt(); + // nocommit -- how to avoid this copy? plus, the copy + // inside BII. if mmapdir how can we directly access + // underlying ram w/ no copy? + in.readBytes(input, 0, numBytes); + intInput.rewind(); + decompressor.decompress(); + } + } + + private static class MMapBlockReader implements FixedIntBlockIndexInput.BlockReader { + private final MMapIndexInput in; + private final int[] buffer; + private final FrameOfRef decompressor; + private final ByteBuffer byteBufferIn; + private final IntBuffer intBufferIn; + + public MMapBlockReader(MMapIndexInput in, int[] buffer) { + this.in = in; + this.buffer = buffer; + + // nocommit -- we seem to be creating this class + // alot... how come? + decompressor = new FrameOfRef(); + + byteBufferIn = in.getBuffer(); + //System.out.println("pos start=" + byteBufferIn.position()); + // nocommit -- this sets pos relative to current pos + // of byteBuffer (confusing) -- this is why we -1 in + // seek (pos is always 4 here) + intBufferIn = byteBufferIn.asIntBuffer(); + + decompressor.setCompressedBuffer(intBufferIn); + decompressor.setUnCompressedData(buffer, 0, buffer.length); + //System.out.println("mmmap"); + } + + public void seek(long pos) throws IOException { + assert pos%4 == 0; + //System.out.println("this=" + this + " seek=" + pos); + intBufferIn.position((int) (pos/4)-1); + } + + public void readBlock() throws IOException { + // nocommit -- we don't need this numBytes header -- + // it's a waste. we need something like the zip + // interface -- the decompressor asks for more bytes + // if it needs it + //int numBytes = in.readInt(); + int numBytes = intBufferIn.get(); + //System.out.println("NB=" + numBytes); + // nocommit -- how to avoid this copy? plus, the copy + // inside BII. if mmapdir how can we directly access + // underlying ram w/ no copy? + //in.readBytes(input, 0, numBytes); + decompressor.decompress(); + } + } + + private static class MultiMMapBlockReader implements FixedIntBlockIndexInput.BlockReader { + private final MultiMMapIndexInput in; + private final int[] buffer; + private final FrameOfRef decompressor; + private final ByteBuffer byteBufferIn; + private final IntBuffer intBufferIn; + + public MultiMMapBlockReader(MultiMMapIndexInput in, int[] buffer) { + this.in = in; + this.buffer = buffer; + + decompressor = new FrameOfRef(); + + byteBufferIn = in.getBuffer(); + intBufferIn = byteBufferIn.asIntBuffer(); + + decompressor.setCompressedBuffer(intBufferIn); + decompressor.setUnCompressedData(buffer, 0, buffer.length); + } + + public void seek(long pos) throws IOException { + // + + } + + public void readBlock() throws IOException { + int numBytes = in.readInt(); + // nocommit -- how to avoid this copy? plus, the copy + // inside BII. if mmapdir how can we directly access + // underlying ram w/ no copy? + //in.readBytes(input, 0, numBytes); + //decompressor.decompress(); + } + } + + protected FixedIntBlockIndexInput.BlockReader getBlockReader(IndexInput in, int[] buffer) { + if (in instanceof MMapIndexInput) { + return new MMapBlockReader((MMapIndexInput) in, buffer); + } else if (false && in instanceof MultiMMapIndexInput) { + return new MultiMMapBlockReader((MultiMMapIndexInput) in, buffer); + } else { + return new BlockReader(in, buffer); + } + } +} + Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (revision 981713) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (working copy) @@ -27,10 +27,12 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.codecs.standard.StandardPostingsReader; import org.apache.lucene.index.codecs.standard.TermState; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.CodecUtil; /** Concrete class that reads the current doc/freq/skip @@ -264,8 +266,11 @@ private boolean omitTF; private boolean storePayloads; private Bits skipDocs; - private final IntIndexInput.Reader docReader; - private final IntIndexInput.Reader freqReader; + // nocommit + //private final IntIndexInput.Reader docReader; + //private final IntIndexInput.Reader freqReader; + private final FixedIntBlockIndexInput.Reader docReader; + private final FixedIntBlockIndexInput.Reader freqReader; private long skipOffset; private final IntIndexInput.Index docIndex; @@ -280,10 +285,15 @@ SepDocsEnum() throws IOException { startDocIn = docIn; - docReader = docIn.reader(); + bulkResult = new BulkReadResult(); + + docReader = (FixedIntBlockIndexInput.Reader) docIn.reader(); + bulkResult.docs.ints = docReader.pending; + docIndex = docIn.index(); if (freqIn != null) { - freqReader = freqIn.reader(); + freqReader = (FixedIntBlockIndexInput.Reader) freqIn.reader(); + bulkResult.freqs.ints = freqReader.pending; freqIndex = freqIn.index(); } else { freqReader = null; @@ -324,6 +334,7 @@ @Override public int nextDoc() throws IOException { + //System.out.println(" SEP: nextDoc count=" + count + " vs df=" + docFreq + " doc=" + doc); while(true) { if (count == docFreq) { @@ -349,7 +360,30 @@ @Override public int read() throws IOException { - // TODO: -- switch to bulk read api in IntIndexInput + final int left = docFreq - count; + final int chunk; + if (left > 128) { + chunk = 128; + } else { + chunk = left; + } + IntsRef d1 = docReader.read(chunk); + IntsRef f1 = freqReader.read(chunk); + assert d1.ints == bulkResult.docs.ints; + assert f1.ints == bulkResult.freqs.ints; + assert f1.length == d1.length; + bulkResult.docs.offset = d1.offset; + bulkResult.freqs.offset = f1.offset; + count += f1.length; + return f1.length; + /* + System.out.println("offset=" + d1.offset + " and " + f1.offset); + for(int i=0;i>> 1; // shift off low bit + if ((code & 1) == 0) { // if low bit is set + freqIn.readVInt(); // skip freq + } + } + + if (skipDocs == null || !skipDocs.get(doc)) { + docs[i] = doc; + ++i; + } + } + + return i; + } + + @Override public int docID() { return doc; } Index: lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java (revision 981713) +++ lucene/src/java/org/apache/lucene/index/codecs/intblock/SimpleIntBlockIndexInput.java (working copy) @@ -52,6 +52,9 @@ this.buffer = buffer; } + public void seek(long pos) { + } + public void readBlock() throws IOException { // silly impl for(int i=0;i // Integer.MAX_VALUE in size using multiple byte buffers. - private class MultiMMapIndexInput extends IndexInput { + // nocommit public + public class MultiMMapIndexInput extends IndexInput { private ByteBuffer[] buffers; private int[] bufSizes; // keep here, ByteBuffer.size() method is optional @@ -320,6 +326,11 @@ seek(0L); } + public ByteBuffer getBuffer() { + // nocommit fixup + return null; + } + @Override public byte readByte() throws IOException { // Performance might be improved by reading ahead into an array of Index: lucene/src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FSDirectory.java (revision 981713) +++ lucene/src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -200,6 +200,7 @@ return new SimpleFSDirectory(path, lockFactory); } else { return new NIOFSDirectory(path, lockFactory); + // return new MMapDirectory(path, lockFactory); } } Index: lucene/src/java/org/apache/lucene/util/BitUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/util/BitUtil.java (revision 981713) +++ lucene/src/java/org/apache/lucene/util/BitUtil.java (working copy) @@ -800,7 +800,7 @@ } /** returns the next highest power of two, or the current value if it's already a power of two or zero*/ - public static long nextHighestPowerOfTwo(long v) { + public static long nextHighestPowerOfTwo(long v) { v--; v |= v >> 1; v |= v >> 2; @@ -812,4 +812,26 @@ return v; } + /** Returns the smallest non negative p such that a given value < (2**(p+1)) + * This differs from (63 - java.lang.Long.numberOfLeadingZeros(v)) + * for non positive given values. + */ + public static int logNextHigherPowerOfTwo(long v) { + long vinput = v; // only for assertions below. + int p = 0; + while (v >= (1 << 8)) { + v >>= 8; + p += 8; + } + while (v >= (1 << 1)) { + v >>= 1; + p++; + } + assert (p <= 62) : p; + assert (p == 62) || (vinput < (1L << (p + 1))) : "p " + p + ", vinput " + vinput; + assert (p == 0) || (vinput >= (1L << p)) : "p " + p + ", vinput " + vinput; + assert (vinput <= 0) || (p == (63 - java.lang.Long.numberOfLeadingZeros(vinput))) : "p " + p + ", vinput " + vinput; + return p; + } + } Index: lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For5Decompress.java (revision 0) @@ -0,0 +1,76 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For5Decompress extends ForDecompress { + static final int numFrameBits = 5; + static final int mask = (int) ((1L<>> 5) & mask; + output[2 + outputOffset] = (intValue0 >>> 10) & mask; + output[3 + outputOffset] = (intValue0 >>> 15) & mask; + output[4 + outputOffset] = (intValue0 >>> 20) & mask; + output[5 + outputOffset] = (intValue0 >>> 25) & mask; + output[6 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[7 + outputOffset] = (intValue1 >>> 3) & mask; + output[8 + outputOffset] = (intValue1 >>> 8) & mask; + output[9 + outputOffset] = (intValue1 >>> 13) & mask; + output[10 + outputOffset] = (intValue1 >>> 18) & mask; + output[11 + outputOffset] = (intValue1 >>> 23) & mask; + output[12 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[13 + outputOffset] = (intValue2 >>> 1) & mask; + output[14 + outputOffset] = (intValue2 >>> 6) & mask; + output[15 + outputOffset] = (intValue2 >>> 11) & mask; + output[16 + outputOffset] = (intValue2 >>> 16) & mask; + output[17 + outputOffset] = (intValue2 >>> 21) & mask; + output[18 + outputOffset] = (intValue2 >>> 26) & mask; + output[19 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; + output[20 + outputOffset] = (intValue3 >>> 4) & mask; + output[21 + outputOffset] = (intValue3 >>> 9) & mask; + output[22 + outputOffset] = (intValue3 >>> 14) & mask; + output[23 + outputOffset] = (intValue3 >>> 19) & mask; + output[24 + outputOffset] = (intValue3 >>> 24) & mask; + output[25 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; + output[26 + outputOffset] = (intValue4 >>> 2) & mask; + output[27 + outputOffset] = (intValue4 >>> 7) & mask; + output[28 + outputOffset] = (intValue4 >>> 12) & mask; + output[29 + outputOffset] = (intValue4 >>> 17) & mask; + output[30 + outputOffset] = (intValue4 >>> 22) & mask; + output[31 + outputOffset] = intValue4 >>> 27; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For7Decompress.java (revision 0) @@ -0,0 +1,78 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For7Decompress extends ForDecompress { + static final int numFrameBits = 7; + static final int mask = (int) ((1L<>> 7) & mask; + output[2 + outputOffset] = (intValue0 >>> 14) & mask; + output[3 + outputOffset] = (intValue0 >>> 21) & mask; + output[4 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; + output[5 + outputOffset] = (intValue1 >>> 3) & mask; + output[6 + outputOffset] = (intValue1 >>> 10) & mask; + output[7 + outputOffset] = (intValue1 >>> 17) & mask; + output[8 + outputOffset] = (intValue1 >>> 24) & mask; + output[9 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[10 + outputOffset] = (intValue2 >>> 6) & mask; + output[11 + outputOffset] = (intValue2 >>> 13) & mask; + output[12 + outputOffset] = (intValue2 >>> 20) & mask; + output[13 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; + output[14 + outputOffset] = (intValue3 >>> 2) & mask; + output[15 + outputOffset] = (intValue3 >>> 9) & mask; + output[16 + outputOffset] = (intValue3 >>> 16) & mask; + output[17 + outputOffset] = (intValue3 >>> 23) & mask; + output[18 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[19 + outputOffset] = (intValue4 >>> 5) & mask; + output[20 + outputOffset] = (intValue4 >>> 12) & mask; + output[21 + outputOffset] = (intValue4 >>> 19) & mask; + output[22 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[23 + outputOffset] = (intValue5 >>> 1) & mask; + output[24 + outputOffset] = (intValue5 >>> 8) & mask; + output[25 + outputOffset] = (intValue5 >>> 15) & mask; + output[26 + outputOffset] = (intValue5 >>> 22) & mask; + output[27 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[28 + outputOffset] = (intValue6 >>> 4) & mask; + output[29 + outputOffset] = (intValue6 >>> 11) & mask; + output[30 + outputOffset] = (intValue6 >>> 18) & mask; + output[31 + outputOffset] = intValue6 >>> 25; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For9Decompress.java (revision 0) @@ -0,0 +1,80 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For9Decompress extends ForDecompress { + static final int numFrameBits = 9; + static final int mask = (int) ((1L<>> 9) & mask; + output[2 + outputOffset] = (intValue0 >>> 18) & mask; + output[3 + outputOffset] = ((intValue0 >>> 27) | (intValue1 << 5)) & mask; + output[4 + outputOffset] = (intValue1 >>> 4) & mask; + output[5 + outputOffset] = (intValue1 >>> 13) & mask; + output[6 + outputOffset] = (intValue1 >>> 22) & mask; + output[7 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[8 + outputOffset] = (intValue2 >>> 8) & mask; + output[9 + outputOffset] = (intValue2 >>> 17) & mask; + output[10 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[11 + outputOffset] = (intValue3 >>> 3) & mask; + output[12 + outputOffset] = (intValue3 >>> 12) & mask; + output[13 + outputOffset] = (intValue3 >>> 21) & mask; + output[14 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[15 + outputOffset] = (intValue4 >>> 7) & mask; + output[16 + outputOffset] = (intValue4 >>> 16) & mask; + output[17 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; + output[18 + outputOffset] = (intValue5 >>> 2) & mask; + output[19 + outputOffset] = (intValue5 >>> 11) & mask; + output[20 + outputOffset] = (intValue5 >>> 20) & mask; + output[21 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[22 + outputOffset] = (intValue6 >>> 6) & mask; + output[23 + outputOffset] = (intValue6 >>> 15) & mask; + output[24 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[25 + outputOffset] = (intValue7 >>> 1) & mask; + output[26 + outputOffset] = (intValue7 >>> 10) & mask; + output[27 + outputOffset] = (intValue7 >>> 19) & mask; + output[28 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[29 + outputOffset] = (intValue8 >>> 5) & mask; + output[30 + outputOffset] = (intValue8 >>> 14) & mask; + output[31 + outputOffset] = intValue8 >>> 23; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/PFor.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/PFor.java (revision 0) @@ -0,0 +1,415 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +import java.util.Arrays; + +/** Patched Frame of Reference PFOR compression/decompression. + *

+ * As defined in:
+ * Super-Scalar RAM-CPU Cache Compression
+ * Marcin Zukowski, Sándor Héman, Niels Nes, Peter Boncz, 2006.
+ * with extensions from:
+ * Performance of Compressed Inverted List Caching in Search Engines
+ * Jiangong Zhang, Xiaohui Long, Torsten Suel, 2008.
+ *

+ * This class does not provide delta coding because the lucene index + * structures already have that. + *

+ * The implementation uses 0 as lower bound for the frame, + * so small positive integers will be most effectively compressed. + *

+ * Some optimized code is used for decompression, + * see class ForDecompress and its subclasses. + *
Good decompression performance will depend on the performance + * of java.nio.IntBuffer indexed get() methods. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * The start point of first exception is at its natural boundary: + * 2 byte exceptions at even byte position, 4 byte at quadruple. + *

+ * To be done: + *

    + *
  • + * Optimize compression code. + *
  • + * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression for to + * use the relative get() method. + *
  • + * Check javadoc generation and generated javadocs. Add javadoc code references. + *
+ */ +public class PFor extends FrameOfRef { + /** Index on input and in compressed frame of first exception, -1 when no exceptions */ + private int firstExceptionIndex; + + /** CHECKME: Move this to another class defining various decompression methods? */ + private final int PFOR_COMPRESSION = 2; /** to encode compression method in header */ + + /** How to encode PFor exceptions: 0: byte, 1: short, 2:int, unused: 3: long */ + private int exceptionCode = -1; + + /** Total number of exception values */ + private int numExceptions; + + /** Create a PFor compressor/decompressor. */ + public PFor() { + } + + /** Compress the decompressed data into the buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored into the buffer, encoding a.o. numFrameBits and unComprSize. + * All ints < 2**numFrameBits are stored sequentially in compressed form + * in the buffer. + * All other ints are stored in the buffer as exceptions after the compressed sequential ints, + * using 1, 2 or 4 bytes per exception, starting at the first byte after the compressed + * sequential ints. + *
+ * The index of the first exception is encoded in the header in the buffer, + * all later exceptions have the offset to the next exception as their value, + * the last one offset to just after the available input size. + * After the first exception, when the next exception index does not fit in + * numFrameBits bits, an exception after 2**numFrameBits inputs is forced and inserted. + *
+ * Exception values are stored in the order of the exceptions. + * The number of bytes used for an exception is also encoded in the header. + * This depends on the maximum exception value and does not vary between the exceptions. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + numExceptions = 0; + int maxException = -1; + firstExceptionIndex = -1; + int lastExceptionIndex = -1; + int i; + int[] exceptionValues = new int[unComprSize]; + int maxNonExceptionMask = (int) ((1L << numFrameBits) - 1); + int maxChain = 254; // maximum value of firstExceptionIndex in header + // CHECKME: maxChain 1 off because of initial value of lastExceptionIndex and force exception test below? + for (i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + // FIXME: split this loop to avoid if statement in loop. + // use predication for this: (someBool ? 1 : 0), and hope that the jit optimizes this. + if ( (((v & maxNonExceptionMask) == v) // no value exception + && (i < (lastExceptionIndex + maxChain)))) { // no forced exception + encodeCompressedValue(i, v); // normal encoding + } else { // exception + exceptionValues[numExceptions] = v; + numExceptions++; + if (firstExceptionIndex == -1) { + firstExceptionIndex = i; + assert firstExceptionIndex <= 254; // maximum value of firstExceptionIndex in header + maxException = v; + maxChain = 1 << ((30 < numFrameBits) ? 30 : numFrameBits); // fewer bits available for exception chain value. + } else if (v > maxException) { + maxException = v; + } + // encode the previous exception pointer + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); + } + lastExceptionIndex = i; + } + } + if (lastExceptionIndex >= 0) { + encodeCompressedValue(lastExceptionIndex, i - lastExceptionIndex - 1); // end the exception chain. + } + int bitsInArray = numFrameBits * unCompressedData.length; + int bytesInArray = (bitsInArray + 7) / 8; + if (maxException < (1 << 8)) { // exceptions as byte + exceptionCode = 0; + } else if (maxException < (1 << 16)) { // exceptions as 2 bytes + exceptionCode = 1; + } else /* if (maxException < (1L << 32)) */ { // exceptions as 4 bytes + exceptionCode = 2; + } + encodeHeader(unComprSize, firstExceptionIndex); + encodeExceptionValues(exceptionValues); + } + + /** Return the number bytes used for a single exception */ + private int exceptionByteSize() { + assert exceptionCode >= 0; + assert exceptionCode <= 2; + return exceptionCode == 0 ? 1 + : exceptionCode == 1 ? 2 + : 4; + } + + /** Return the number of exceptions. + * Only valid after compress() or decompress(). + */ + public int getNumExceptions() { + return numExceptions; + } + + private int compressedArrayByteSize() { + int compressedArrayBits = unComprSize * numFrameBits; + return (compressedArrayBits + 7) / 8; + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + // numExceptions only valid after compress() or decompress() + return HEADER_SIZE + + ((compressedArrayByteSize() + + exceptionByteSize() * numExceptions + + 3) >> 2); // round up to next multiple of 4 and divide by 4 + } + + private void encodeExceptionValues(int[] exceptionValues) { + if ((compressedBuffer == null) || (numExceptions == 0)) { + return; + } + int excByteOffset = compressedArrayByteSize(); + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + int i = 0; + do { + int intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); // round down here. + setBufferIntBits(intIndex, ((excByteOffset & 3) * 8), 8, exceptionValues[i]); + excByteOffset++; + } while (++i < numExceptions); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + int intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { // encode first 2 byte exception in high 2 bytes of same int as last frame bits. + setBufferIntBits(intIndex, 16, 16, exceptionValues[i]); + intIndex++; + i++; + } + for (; i < (numExceptions-1); i += 2) { + compressedBuffer.put(intIndex++, (exceptionValues[i+1] << 16) | exceptionValues[i]); + } + if (i < numExceptions) { + compressedBuffer.put(intIndex, exceptionValues[i]); // also clear the high 16 bits + } + } + break; + + case 2: { // 4 byte exceptions + int excIntOffSet = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + int i = 0; + do { + compressedBuffer.put(excIntOffSet + i, exceptionValues[i]); + } while(++i < numExceptions); + } + break; + } + } + + /** Decode the exception values while going through the exception chain. + *
For performance, delegate/subclass this to classes with fixed exceptionCode. + *
Also, decoding exceptions is preferably done from an int border instead of + * from a random byte directly after the compressed array. This will allow faster + * decoding of exceptions, at the cost of at most 3 bytes. + *
When ((numFrameBits * unComprSize) % 32) == 0, this cost will always be + * zero bytes so specialize for these cases. + */ + private void patchExceptions() { + numExceptions = 0; + if (firstExceptionIndex == -1) { + return; + } + int excIndex = firstExceptionIndex; + int excByteOffset = compressedArrayByteSize(); + int excValue; + int intIndex; + + switch (exceptionCode) { + case 0: { // 1 byte exceptions + do { + intIndex = COMPRESSED_INDEX + (excByteOffset >> 2); + int firstBitPosition = (excByteOffset & 3) << 3; + excValue = (compressedBuffer.get(intIndex) >>> firstBitPosition) & ((1 << 8) - 1); + excIndex = patch(excIndex, excValue); + excByteOffset++; + } while (excIndex < unComprSize); + } + break; + + case 1: { // 2 byte exceptions + int excShortOffset = (excByteOffset + 1) >> 1; // to next multiple of two bytes. + intIndex = COMPRESSED_INDEX + (excShortOffset >> 1); // round down here. + int i = 0; + if ((excShortOffset & 1) != 0) { + // decode first 2 byte exception from high 2 bytes of same int as last frame bits. + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + while (excIndex < unComprSize) { + excValue = compressedBuffer.get(intIndex) & ((1<<16)-1); + excIndex = patch(excIndex, excValue); + if (excIndex >= unComprSize) { + break; + } + excValue = compressedBuffer.get(intIndex++) >>> 16; + excIndex = patch(excIndex, excValue); + } + } + break; + + case 2: // 4 byte exceptions + intIndex = COMPRESSED_INDEX + ((excByteOffset + 3) >> 2); // to next multiple of four bytes, in ints. + do { + excValue = compressedBuffer.get(intIndex++); + excIndex = patch(excIndex, excValue); + } while (excIndex < unComprSize); + break; + } + } + + /** The 4 byte header (32 bits) contains: + * + * - 4 bits for the compression method: 0b0001 for PFor + * - 4 bits unused + * + * - 5 bits for (numFrameBits-1) + * - 2 bits for the exception code: 0b00: byte, 0b01: short, 0b10: int, 0b11: long (unused). + * - 1 bit unused + * + * - 8 bits for uncompressed input size - 1, + * + * - 8 bits for the index of the first exception + 1, (0 when no exceptions) + */ + private void encodeHeader(int unComprSize, int firstExceptionIndex) { + assert exceptionCode >= 0; + assert exceptionCode <= 2; // 3 for long, but unused for now. + assert numFrameBits >= 1; + assert numFrameBits <= 32; + assert unComprSize >= 1; + assert unComprSize <= 128; + assert firstExceptionIndex >= -1; + assert firstExceptionIndex < unComprSize; + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((firstExceptionIndex+1) << 24) + | ((unComprSize-1) << 16) + | ((exceptionCode & 3) << 13) | ((numFrameBits-1) << 8) + | (PFOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(); + firstExceptionIndex = ((header >>> 24) & 255) - 1; + //unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + assert numFrameBits > 0: numFrameBits; + assert numFrameBits <= 32: numFrameBits; + // verify compression method: + assert PFOR_COMPRESSION == ((header >>> 4) & 15); + exceptionCode = (header >>> 13) & 3; + assert exceptionCode <= 2; + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + super.decompress(); + patchExceptions(); + } + + /** Patch and return index of next exception */ + private int patch(int excIndex, int excValue) { + int nextExceptionIndex = unCompressedData[excIndex] + excIndex + 1; // chain offset + unCompressedData[excIndex + offset] = excValue; // patch + assert nextExceptionIndex > excIndex; + numExceptions++; + return nextExceptionIndex; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * This is done by taking a copy of the input, sorting it and using this + * to determine the compressed size for each possible numbits in a single pass, + * ignoring forced exceptions. + * Finally an estimation of the number of forced exceptions is reduced to + * less than 1 in 32 input numbers by increasing the number of frame bits. + * This implementation works by determining the total number of bytes needed for + * the compressed data, but does take into account alignment of exceptions + * at 2 or 4 byte boundaries. + */ + public int frameBitsForCompression() { + if ((offset + unComprSize) > unCompressedData.length) { + throw new IllegalArgumentException( "(offset " + offset + + " + unComprSize " + unComprSize + + ") > unCompressedData.length " + unCompressedData.length); + } + int copy[] = Arrays.copyOfRange(unCompressedData, offset, offset + unComprSize); + assert copy.length == unComprSize; + Arrays.sort(copy); + int maxValue = copy[copy.length-1]; + if (maxValue <= 1) { + return 1; + } + int bytesPerException = (maxValue < (1 << 8)) ? 1 : (maxValue < (1 << 16)) ? 2 : 4; + int frameBits = 1; + int bytesForFrame = (copy.length * frameBits + 7) / 8; + // initially assume all input is an exception. + int totalBytes = bytesForFrame + copy.length * bytesPerException; // excluding the header. + int bestBytes = totalBytes; + int bestFrameBits = frameBits; + int bestExceptions = copy.length; + for (int i = 0; i < copy.length; i++) { + // determine frameBits so that copy[i] is no more exception + while (copy[i] >= (1 << frameBits)) { + if (frameBits == 30) { // no point to increase further. + return bestFrameBits; + } + ++frameBits; + // increase bytesForFrame and totalBytes to correspond to frameBits + int newBytesForFrame = (copy.length * frameBits + 7) / 8; + totalBytes += newBytesForFrame - bytesForFrame; + bytesForFrame = newBytesForFrame; + } + totalBytes -= bytesPerException; // no more need to store copy[i] as exception + if (totalBytes <= bestBytes) { // <= : prefer fewer exceptions at higher number of frame bits. + bestBytes = totalBytes; + bestFrameBits = frameBits; + bestExceptions = (copy.length - i - 1); + } + } + if (bestExceptions > 0) { // check for forced exceptions. + // This ignores the position of the first exception, for which enough bits are available in the header. + int allowedNumExceptions = bestExceptions + (copy.length >> 5); // 1 in 32 is allowed to be forced. + // (copy.length >> bestFrameBits): Minimum exception chain size including forced ones, + // ignoring the position of the first exception. + while (allowedNumExceptions < (copy.length >> bestFrameBits)) { // Too many forced? + bestFrameBits++; // Reduce forced exceptions and perhaps reduce actual exceptions + // Dilemma: decompression speed reduces with increasing number of frame bits, + // so it may be better to increase no more than once or twice here. + } + } + return bestFrameBits; + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/ForDecompress.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; + +/** PFor frame decompression for any number of frame bits. */ +class ForDecompress { + + static void decodeAnyFrame( + IntBuffer intBuffer, int bufIndex, int inputSize, int numFrameBits, + int[] output, int outputOffset) { + + assert numFrameBits > 0 : numFrameBits; + assert numFrameBits <= 32 : numFrameBits; + final int mask = (int) ((1L<>> bitPos) & mask; + if (--inputSize == 0) return; + bitPos += numFrameBits; + } + + int intValue2 = intBuffer.get(++bufIndex); + output[++outputOffset] = ( (bitPos == 32) + ? intValue2 + : ((intValue1 >>> bitPos) | (intValue2 << (32 - bitPos))) + ) & mask; + + if (--inputSize == 0) return; + + intValue1 = intValue2; + bitPos += numFrameBits - 32; + } while (true); + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/FrameOfRef.java (revision 0) @@ -0,0 +1,327 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +import java.util.Arrays; + +import org.apache.lucene.util.BitUtil; + +/** Frame of Reference lossless integer compression/decompression. + * For positive integers, the compression is done by leaving out + * the most significant bits, and storing all numbers with a fixed number of bits + * contiguously in a buffer of bits. This buffer is called the frame, and it + * can store positive numbers in a range from 0 to a constant maximum fitting in + * the number of bits available for a single compressed number. + *

+ * This implementation uses 0 as the lower bound reference for the frame, + * so small positive integers can be most effectively compressed. + *

+ * Optimized code is used for decompression, see class ForDecompress and its subclasses. + *
Use of the -server option helps performance for the Sun 1.6 jvm under Linux. + *

+ * This class does not provide delta coding because the Lucene index + * structures already have that. + *

+ * To be done: + *

    + *
  • + * Optimize compression code by specializing for number of frame bits. + *
  • + * IntBuffer.get() is somewhat faster that IntBuffer.get(index), adapt (de)compression to + * use the relative get() method. + *
  • + * Check javadoc generation and generated javadocs. Add javadoc code references. + *
+ */ +public class FrameOfRef { + /** Number of frame bits. 2**numFrameBits - 1 is the maximum compressed value. */ + protected int numFrameBits; + + /** Constant header tag to allow other compression methods, use value 0001 for + * Frame of reference. + * CHECKME: Move this to another class defining various decompression methods? + */ + //protected int compressionMethod; + private final int FOR_COMPRESSION = 1; /** encode compression method in header */ + + /** IntBuffer for compressed data */ + protected IntBuffer compressedBuffer; + + /** Index of header in int buffer */ + protected final int HEADER_INDEX = 0; + + /** Start index in int buffer of array integers each compressed to numFrameBits. */ + protected final int COMPRESSED_INDEX = HEADER_INDEX + 1; + protected final int HEADER_SIZE = 1; // one integer in IntBuffer + + // used by decompress to read the "input" (compressed) ints + protected int[] scratch = new int[32]; + + /** Uncompressed data */ + protected int[] unCompressedData; + /** Offset into unCompressedData */ + protected int offset; + /** Size of unCompressedData, -1 when not available. */ + protected int unComprSize = -1; + + /** Create a Frame of Reference integer compressor/decompressor. */ + public FrameOfRef() { + } + + /** Integer buffer to hold the compressed data.
+ * Compression and decompression do not affect the current buffer position, + * and the beginning of the compressed data should be or will be at the current + * buffer position.
+ * When the buffer is not large enough, ArrayIndexOutOfBoundExceptions will occur + * during compression/decompression.
+ * Without a buffer for compressed data, compress() will only determine the number + * of integers needed in the buffer, see compress().
+ * Without a valid buffer, decompress() will throw a NullPointerException.
+ * For optimal speed when the IntBuffer is a view on a ByteBuffer, + * the IntBuffer should have a byte offset of a multiple of 4 bytes, possibly 0.
+ * An IntBuffer is used here because 32 bits can efficiently accessed in the buffer + * on all current processors, and a positive int is normally large enough + * for various purposes in a Lucene index. + * + * @param compressedBuffer The buffer to hold the compressed integers. + * + */ + public void setCompressedBuffer(IntBuffer compressedBuffer) { + this.compressedBuffer = compressedBuffer; + } + + + /** Array with offset holding uncompressed data. + * @param unCompressedData The array holding uncompressed integers. + * @param offset offset in unCompressedData. + * @param unComprSize The number of uncompressed integers, should be at least 1. + */ + public void setUnCompressedData(int[] unCompressedData, int offset, int unComprSize) { + assert unCompressedData != null; + assert offset >= 0; + assert unComprSize >= 1; + assert (offset + unComprSize) <= unCompressedData.length; + this.unCompressedData = unCompressedData; + this.offset = offset; + this.unComprSize = unComprSize; + } + + /** Compress the uncompressed data into the buffer using the given number of + * frame bits, storing only this number of least significant bits of the + * uncompressed integers in the compressed buffer. + * Should only be used after setUnCompressedData(). + *
+ * When setCompressBuffer() was not done, no actual compression is done. + * Regardless of the use of setCompressBuffer(), bufferByteSize() will return + * a valid value after calling compress(). + *

+ * When a buffer is available, the following is done. + * A header is stored as a first integer into the buffer, encoding + * the compression method, the number of frame bits and the number of compressed integers. + * All uncompressed integers are stored sequentially in compressed form + * in the buffer after the header. + * + * @param numFrameBits The number of frame bits. Should be between 1 and 32. + * Note that when this value is 32, no compression occurs. + */ + public void compress(int numFrameBits) { + assert numFrameBits >= 1; + assert numFrameBits <= 32; + this.numFrameBits = numFrameBits; + encodeHeader(unComprSize); + for (int i = 0; i < unComprSize; i++) { + int v = unCompressedData[i + offset]; + encodeCompressedValue(i, v); + } + } + + /** As compress(), using the result of frameBitsForCompression() as the number of frame bits. */ + public void compress() { + compress( frameBitsForCompression()); + } + + /** Return the number of integers used in IntBuffer. + * Only valid after compress() or decompress(). + */ + public int compressedSize() { + return HEADER_SIZE + (unComprSize * numFrameBits + 31) / 32; + } + + /** Encode an integer value by compressing it into the buffer. + * @param compressedPos The index of the compressed integer in the compressed buffer. + * @param value The non negative value to be stored in compressed form. + * This should fit into the number of frame bits. + */ + protected void encodeCompressedValue(int compressedPos, int value) { + encodeCompressedValueBase(compressedPos, value, numFrameBits); // FIXME: inline private method. + } + + /** Encode a value into the compressed buffer. + * Since numFrameBits is always smaller than the number of bits in an int, + * at most two ints in the buffer will be affected. + *
Has no effect when compressedBuffer == null. + *
This could be specialized for numBits just like decompressFrame(). + */ + private void encodeCompressedValueBase(int compressedPos, int value, int numBits) { + assert numBits >= 1; + assert numBits <= 32; + if (compressedBuffer == null) { + return; + } + final int mask = (int) ((1L << numBits) - 1); + assert ((value & mask) == value) : ("value " + value + ", mask " + mask + ", numBits " + numBits); // lossless compression + final int compressedBitPos = numBits * compressedPos; + final int firstBitPosition = compressedBitPos & 31; + int intIndex = COMPRESSED_INDEX + (compressedBitPos >> 5); + setBufferIntBits(intIndex, firstBitPosition, numBits, value); + if ((firstBitPosition + numBits) > 32) { // value does not fit in first int + setBufferIntBits(intIndex+1, 0, (firstBitPosition + numBits - 32), (value >>> (32 - firstBitPosition))); + } + } + + /** Change bits of an integer in the compressed buffer. + *
A more efficient implementation is possible when the compressed + * buffer is known to contain only zero bits, in that case one mask operation can be removed. + * @param intIndex The index of the affected integer in the compressed buffer. + * @param firstBitPosition The position of the least significant bit to be changed. + * @param numBits The number of more significant bits to be changed. + * @param value The new value of the bits to be changed, with the least significant bit at position zero. + */ + protected void setBufferIntBits(int intIndex, int firstBitPosition, int numBits, int value) { + final int mask = (int) ((1L << numBits) - 1); + compressedBuffer.put(intIndex, + (compressedBuffer.get(intIndex) + & ~ (mask << firstBitPosition)) // masking superfluous on clear buffer + | (value << firstBitPosition)); + } + + /** The 4 byte header (32 bits) contains: + *

    + *
  • + *
      + *
    • 4 bits for the compression method: 0b0001 for FrameOfRef, + *
    • 4 bits unused, + *
    + *
  • + *
      + *
    • 5 bits for (numFrameBits-1), + *
    • 3 bit unused, + *
    + *
  • 8 bits for number of compressed integers - 1, + *
  • 8 bits unused. + *
+ */ + private void encodeHeader(int unComprSize) { + assert numFrameBits >= 1; + assert numFrameBits <= (1 << 5); // 32 + assert unComprSize >= 1; + assert unComprSize <= (1 << 8); // 256 + if (compressedBuffer != null) { + compressedBuffer.put(HEADER_INDEX, + ((unComprSize-1) << 16) + | ((numFrameBits-1) << 8) + | (FOR_COMPRESSION << 4)); + } + } + + protected void decodeHeader() { + int header = compressedBuffer.get(); + // nocommit -- we know this will always be fixed (eg + // 128)? silly to encode in every frame? + //unComprSize = ((header >>> 16) & 255) + 1; + numFrameBits = ((header >>> 8) & 31) + 1; + // verify compression method: + assert FOR_COMPRESSION == ((header >>> 4) & 15); + } + + /** Decompress from the buffer into output from a given offset. */ + public void decompress() { + decodeHeader(); + decompressFrame(); + } + + /** Return the number of integers available for decompression. + * Do not use before an IntBuffer was passed to setCompressBuffer. + */ + public int decompressedSize() { + decodeHeader(); + return unComprSize; + } + + /** For performance, this delegates to classes with fixed numFrameBits. */ + private void decompressFrame() { + switch (numFrameBits) { + // CHECKME: two other implementations might be faster: + // - array of static methods: Method[numFrameBits].invoke(null, [this]), + // - array of non static decompressors: ForDecompressor[numFrameBits].decompressFrame(this) . + case 1: For1Decompress.decompressFrame(this); break; + case 2: For2Decompress.decompressFrame(this); break; + case 3: For3Decompress.decompressFrame(this); break; + case 4: For4Decompress.decompressFrame(this); break; + case 5: For5Decompress.decompressFrame(this); break; + case 6: For6Decompress.decompressFrame(this); break; + case 7: For7Decompress.decompressFrame(this); break; + case 8: For8Decompress.decompressFrame(this); break; + case 9: For9Decompress.decompressFrame(this); break; + case 10: For10Decompress.decompressFrame(this); break; + case 11: For11Decompress.decompressFrame(this); break; + case 12: For12Decompress.decompressFrame(this); break; + case 13: For13Decompress.decompressFrame(this); break; + case 14: For14Decompress.decompressFrame(this); break; + case 15: For15Decompress.decompressFrame(this); break; + case 16: For16Decompress.decompressFrame(this); break; + case 17: For17Decompress.decompressFrame(this); break; + case 18: For18Decompress.decompressFrame(this); break; + case 19: For19Decompress.decompressFrame(this); break; + case 20: For20Decompress.decompressFrame(this); break; + case 21: For21Decompress.decompressFrame(this); break; + case 22: For22Decompress.decompressFrame(this); break; + case 23: For23Decompress.decompressFrame(this); break; + case 24: For24Decompress.decompressFrame(this); break; + case 25: For25Decompress.decompressFrame(this); break; + case 26: For26Decompress.decompressFrame(this); break; + case 27: For27Decompress.decompressFrame(this); break; + case 28: For28Decompress.decompressFrame(this); break; + case 29: For29Decompress.decompressFrame(this); break; + case 30: For30Decompress.decompressFrame(this); break; + case 31: For31Decompress.decompressFrame(this); break; + case 32: For32Decompress.decompressFrame(this); break; + default: + throw new IllegalStateException("Unknown number of frame bits " + numFrameBits); + } + } + + public int getNumFrameBits() { + return numFrameBits; + } + + /** Determine the number of frame bits to be used for compression. + * Use only after setUnCompressedData(). + * @return The number of bits needed to encode the maximum positive uncompressed value. + * Negative uncompressed values have no influence on the result. + */ + public int frameBitsForCompression() { + int maxNonNegVal = 0; + for (int i = offset; i < (offset + unComprSize); i++) { + if (unCompressedData[i] > maxNonNegVal) { + maxNonNegVal = unCompressedData[i]; + } + } + return BitUtil.logNextHigherPowerOfTwo(maxNonNegVal) + 1; + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For11Decompress.java (revision 0) @@ -0,0 +1,82 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For11Decompress extends ForDecompress { + static final int numFrameBits = 11; + static final int mask = (int) ((1L<>> 11) & mask; + output[2 + outputOffset] = ((intValue0 >>> 22) | (intValue1 << 10)) & mask; + output[3 + outputOffset] = (intValue1 >>> 1) & mask; + output[4 + outputOffset] = (intValue1 >>> 12) & mask; + output[5 + outputOffset] = ((intValue1 >>> 23) | (intValue2 << 9)) & mask; + output[6 + outputOffset] = (intValue2 >>> 2) & mask; + output[7 + outputOffset] = (intValue2 >>> 13) & mask; + output[8 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; + output[9 + outputOffset] = (intValue3 >>> 3) & mask; + output[10 + outputOffset] = (intValue3 >>> 14) & mask; + output[11 + outputOffset] = ((intValue3 >>> 25) | (intValue4 << 7)) & mask; + output[12 + outputOffset] = (intValue4 >>> 4) & mask; + output[13 + outputOffset] = (intValue4 >>> 15) & mask; + output[14 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[15 + outputOffset] = (intValue5 >>> 5) & mask; + output[16 + outputOffset] = (intValue5 >>> 16) & mask; + output[17 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; + output[18 + outputOffset] = (intValue6 >>> 6) & mask; + output[19 + outputOffset] = (intValue6 >>> 17) & mask; + output[20 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[21 + outputOffset] = (intValue7 >>> 7) & mask; + output[22 + outputOffset] = (intValue7 >>> 18) & mask; + output[23 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; + output[24 + outputOffset] = (intValue8 >>> 8) & mask; + output[25 + outputOffset] = (intValue8 >>> 19) & mask; + output[26 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[27 + outputOffset] = (intValue9 >>> 9) & mask; + output[28 + outputOffset] = (intValue9 >>> 20) & mask; + output[29 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; + output[30 + outputOffset] = (intValue10 >>> 10) & mask; + output[31 + outputOffset] = intValue10 >>> 21; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For21Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For21Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For21Decompress.java (revision 0) @@ -0,0 +1,92 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For21Decompress extends ForDecompress { + static final int numFrameBits = 21; + static final int mask = (int) ((1L<>> 21) | (intValue1 << 11)) & mask; + output[2 + outputOffset] = (intValue1 >>> 10) & mask; + output[3 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[4 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[5 + outputOffset] = (intValue3 >>> 9) & mask; + output[6 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[7 + outputOffset] = ((intValue4 >>> 19) | (intValue5 << 13)) & mask; + output[8 + outputOffset] = (intValue5 >>> 8) & mask; + output[9 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[10 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[11 + outputOffset] = (intValue7 >>> 7) & mask; + output[12 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[13 + outputOffset] = ((intValue8 >>> 17) | (intValue9 << 15)) & mask; + output[14 + outputOffset] = (intValue9 >>> 6) & mask; + output[15 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; + output[16 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[17 + outputOffset] = (intValue11 >>> 5) & mask; + output[18 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[19 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; + output[20 + outputOffset] = (intValue13 >>> 4) & mask; + output[21 + outputOffset] = ((intValue13 >>> 25) | (intValue14 << 7)) & mask; + output[22 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; + output[23 + outputOffset] = (intValue15 >>> 3) & mask; + output[24 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[25 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; + output[26 + outputOffset] = (intValue17 >>> 2) & mask; + output[27 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; + output[28 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; + output[29 + outputOffset] = (intValue19 >>> 1) & mask; + output[30 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; + output[31 + outputOffset] = intValue20 >>> 11; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For31Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For31Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For31Decompress.java (revision 0) @@ -0,0 +1,102 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For31Decompress extends ForDecompress { + static final int numFrameBits = 31; + static final int mask = (int) ((1L<>> 31) | (intValue1 << 1)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 30) | (intValue2 << 2)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 29) | (intValue3 << 3)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 28) | (intValue4 << 4)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 27) | (intValue5 << 5)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 26) | (intValue6 << 6)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 25) | (intValue7 << 7)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 24) | (intValue8 << 8)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 23) | (intValue9 << 9)) & mask; + output[10 + outputOffset] = ((intValue9 >>> 22) | (intValue10 << 10)) & mask; + output[11 + outputOffset] = ((intValue10 >>> 21) | (intValue11 << 11)) & mask; + output[12 + outputOffset] = ((intValue11 >>> 20) | (intValue12 << 12)) & mask; + output[13 + outputOffset] = ((intValue12 >>> 19) | (intValue13 << 13)) & mask; + output[14 + outputOffset] = ((intValue13 >>> 18) | (intValue14 << 14)) & mask; + output[15 + outputOffset] = ((intValue14 >>> 17) | (intValue15 << 15)) & mask; + output[16 + outputOffset] = ((intValue15 >>> 16) | (intValue16 << 16)) & mask; + output[17 + outputOffset] = ((intValue16 >>> 15) | (intValue17 << 17)) & mask; + output[18 + outputOffset] = ((intValue17 >>> 14) | (intValue18 << 18)) & mask; + output[19 + outputOffset] = ((intValue18 >>> 13) | (intValue19 << 19)) & mask; + output[20 + outputOffset] = ((intValue19 >>> 12) | (intValue20 << 20)) & mask; + output[21 + outputOffset] = ((intValue20 >>> 11) | (intValue21 << 21)) & mask; + output[22 + outputOffset] = ((intValue21 >>> 10) | (intValue22 << 22)) & mask; + output[23 + outputOffset] = ((intValue22 >>> 9) | (intValue23 << 23)) & mask; + output[24 + outputOffset] = ((intValue23 >>> 8) | (intValue24 << 24)) & mask; + output[25 + outputOffset] = ((intValue24 >>> 7) | (intValue25 << 25)) & mask; + output[26 + outputOffset] = ((intValue25 >>> 6) | (intValue26 << 26)) & mask; + output[27 + outputOffset] = ((intValue26 >>> 5) | (intValue27 << 27)) & mask; + output[28 + outputOffset] = ((intValue27 >>> 4) | (intValue28 << 28)) & mask; + output[29 + outputOffset] = ((intValue28 >>> 3) | (intValue29 << 29)) & mask; + output[30 + outputOffset] = ((intValue29 >>> 2) | (intValue30 << 30)) & mask; + output[31 + outputOffset] = intValue30 >>> 1; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For13Decompress.java (revision 0) @@ -0,0 +1,84 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For13Decompress extends ForDecompress { + static final int numFrameBits = 13; + static final int mask = (int) ((1L<>> 13) & mask; + output[2 + outputOffset] = ((intValue0 >>> 26) | (intValue1 << 6)) & mask; + output[3 + outputOffset] = (intValue1 >>> 7) & mask; + output[4 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; + output[5 + outputOffset] = (intValue2 >>> 1) & mask; + output[6 + outputOffset] = (intValue2 >>> 14) & mask; + output[7 + outputOffset] = ((intValue2 >>> 27) | (intValue3 << 5)) & mask; + output[8 + outputOffset] = (intValue3 >>> 8) & mask; + output[9 + outputOffset] = ((intValue3 >>> 21) | (intValue4 << 11)) & mask; + output[10 + outputOffset] = (intValue4 >>> 2) & mask; + output[11 + outputOffset] = (intValue4 >>> 15) & mask; + output[12 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[13 + outputOffset] = (intValue5 >>> 9) & mask; + output[14 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[15 + outputOffset] = (intValue6 >>> 3) & mask; + output[16 + outputOffset] = (intValue6 >>> 16) & mask; + output[17 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; + output[18 + outputOffset] = (intValue7 >>> 10) & mask; + output[19 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; + output[20 + outputOffset] = (intValue8 >>> 4) & mask; + output[21 + outputOffset] = (intValue8 >>> 17) & mask; + output[22 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[23 + outputOffset] = (intValue9 >>> 11) & mask; + output[24 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[25 + outputOffset] = (intValue10 >>> 5) & mask; + output[26 + outputOffset] = (intValue10 >>> 18) & mask; + output[27 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; + output[28 + outputOffset] = (intValue11 >>> 12) & mask; + output[29 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; + output[30 + outputOffset] = (intValue12 >>> 6) & mask; + output[31 + outputOffset] = intValue12 >>> 19; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For23Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For23Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For23Decompress.java (revision 0) @@ -0,0 +1,94 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For23Decompress extends ForDecompress { + static final int numFrameBits = 23; + static final int mask = (int) ((1L<>> 23) | (intValue1 << 9)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 14) | (intValue2 << 18)) & mask; + output[3 + outputOffset] = (intValue2 >>> 5) & mask; + output[4 + outputOffset] = ((intValue2 >>> 28) | (intValue3 << 4)) & mask; + output[5 + outputOffset] = ((intValue3 >>> 19) | (intValue4 << 13)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 10) | (intValue5 << 22)) & mask; + output[7 + outputOffset] = (intValue5 >>> 1) & mask; + output[8 + outputOffset] = ((intValue5 >>> 24) | (intValue6 << 8)) & mask; + output[9 + outputOffset] = ((intValue6 >>> 15) | (intValue7 << 17)) & mask; + output[10 + outputOffset] = (intValue7 >>> 6) & mask; + output[11 + outputOffset] = ((intValue7 >>> 29) | (intValue8 << 3)) & mask; + output[12 + outputOffset] = ((intValue8 >>> 20) | (intValue9 << 12)) & mask; + output[13 + outputOffset] = ((intValue9 >>> 11) | (intValue10 << 21)) & mask; + output[14 + outputOffset] = (intValue10 >>> 2) & mask; + output[15 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; + output[16 + outputOffset] = ((intValue11 >>> 16) | (intValue12 << 16)) & mask; + output[17 + outputOffset] = (intValue12 >>> 7) & mask; + output[18 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; + output[19 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; + output[20 + outputOffset] = ((intValue14 >>> 12) | (intValue15 << 20)) & mask; + output[21 + outputOffset] = (intValue15 >>> 3) & mask; + output[22 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; + output[23 + outputOffset] = ((intValue16 >>> 17) | (intValue17 << 15)) & mask; + output[24 + outputOffset] = (intValue17 >>> 8) & mask; + output[25 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; + output[26 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; + output[27 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; + output[28 + outputOffset] = (intValue20 >>> 4) & mask; + output[29 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; + output[30 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; + output[31 + outputOffset] = intValue22 >>> 9; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For15Decompress.java (revision 0) @@ -0,0 +1,86 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For15Decompress extends ForDecompress { + static final int numFrameBits = 15; + static final int mask = (int) ((1L<>> 15) & mask; + output[2 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[3 + outputOffset] = (intValue1 >>> 13) & mask; + output[4 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[5 + outputOffset] = (intValue2 >>> 11) & mask; + output[6 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[7 + outputOffset] = (intValue3 >>> 9) & mask; + output[8 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[9 + outputOffset] = (intValue4 >>> 7) & mask; + output[10 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[11 + outputOffset] = (intValue5 >>> 5) & mask; + output[12 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[13 + outputOffset] = (intValue6 >>> 3) & mask; + output[14 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[15 + outputOffset] = (intValue7 >>> 1) & mask; + output[16 + outputOffset] = (intValue7 >>> 16) & mask; + output[17 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; + output[18 + outputOffset] = (intValue8 >>> 14) & mask; + output[19 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; + output[20 + outputOffset] = (intValue9 >>> 12) & mask; + output[21 + outputOffset] = ((intValue9 >>> 27) | (intValue10 << 5)) & mask; + output[22 + outputOffset] = (intValue10 >>> 10) & mask; + output[23 + outputOffset] = ((intValue10 >>> 25) | (intValue11 << 7)) & mask; + output[24 + outputOffset] = (intValue11 >>> 8) & mask; + output[25 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; + output[26 + outputOffset] = (intValue12 >>> 6) & mask; + output[27 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; + output[28 + outputOffset] = (intValue13 >>> 4) & mask; + output[29 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; + output[30 + outputOffset] = (intValue14 >>> 2) & mask; + output[31 + outputOffset] = intValue14 >>> 17; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For25Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For25Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For25Decompress.java (revision 0) @@ -0,0 +1,96 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For25Decompress extends ForDecompress { + static final int numFrameBits = 25; + static final int mask = (int) ((1L<>> 25) | (intValue1 << 7)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 18) | (intValue2 << 14)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 11) | (intValue3 << 21)) & mask; + output[4 + outputOffset] = (intValue3 >>> 4) & mask; + output[5 + outputOffset] = ((intValue3 >>> 29) | (intValue4 << 3)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[7 + outputOffset] = ((intValue5 >>> 15) | (intValue6 << 17)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 8) | (intValue7 << 24)) & mask; + output[9 + outputOffset] = (intValue7 >>> 1) & mask; + output[10 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; + output[11 + outputOffset] = ((intValue8 >>> 19) | (intValue9 << 13)) & mask; + output[12 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; + output[13 + outputOffset] = (intValue10 >>> 5) & mask; + output[14 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; + output[15 + outputOffset] = ((intValue11 >>> 23) | (intValue12 << 9)) & mask; + output[16 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; + output[17 + outputOffset] = ((intValue13 >>> 9) | (intValue14 << 23)) & mask; + output[18 + outputOffset] = (intValue14 >>> 2) & mask; + output[19 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; + output[20 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; + output[21 + outputOffset] = ((intValue16 >>> 13) | (intValue17 << 19)) & mask; + output[22 + outputOffset] = (intValue17 >>> 6) & mask; + output[23 + outputOffset] = ((intValue17 >>> 31) | (intValue18 << 1)) & mask; + output[24 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[25 + outputOffset] = ((intValue19 >>> 17) | (intValue20 << 15)) & mask; + output[26 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; + output[27 + outputOffset] = (intValue21 >>> 3) & mask; + output[28 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; + output[29 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; + output[30 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; + output[31 + outputOffset] = intValue24 >>> 7; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For17Decompress.java (revision 0) @@ -0,0 +1,88 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For17Decompress extends ForDecompress { + static final int numFrameBits = 17; + static final int mask = (int) ((1L<>> 17) | (intValue1 << 15)) & mask; + output[2 + outputOffset] = (intValue1 >>> 2) & mask; + output[3 + outputOffset] = ((intValue1 >>> 19) | (intValue2 << 13)) & mask; + output[4 + outputOffset] = (intValue2 >>> 4) & mask; + output[5 + outputOffset] = ((intValue2 >>> 21) | (intValue3 << 11)) & mask; + output[6 + outputOffset] = (intValue3 >>> 6) & mask; + output[7 + outputOffset] = ((intValue3 >>> 23) | (intValue4 << 9)) & mask; + output[8 + outputOffset] = (intValue4 >>> 8) & mask; + output[9 + outputOffset] = ((intValue4 >>> 25) | (intValue5 << 7)) & mask; + output[10 + outputOffset] = (intValue5 >>> 10) & mask; + output[11 + outputOffset] = ((intValue5 >>> 27) | (intValue6 << 5)) & mask; + output[12 + outputOffset] = (intValue6 >>> 12) & mask; + output[13 + outputOffset] = ((intValue6 >>> 29) | (intValue7 << 3)) & mask; + output[14 + outputOffset] = (intValue7 >>> 14) & mask; + output[15 + outputOffset] = ((intValue7 >>> 31) | (intValue8 << 1)) & mask; + output[16 + outputOffset] = ((intValue8 >>> 16) | (intValue9 << 16)) & mask; + output[17 + outputOffset] = (intValue9 >>> 1) & mask; + output[18 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; + output[19 + outputOffset] = (intValue10 >>> 3) & mask; + output[20 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; + output[21 + outputOffset] = (intValue11 >>> 5) & mask; + output[22 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; + output[23 + outputOffset] = (intValue12 >>> 7) & mask; + output[24 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; + output[25 + outputOffset] = (intValue13 >>> 9) & mask; + output[26 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; + output[27 + outputOffset] = (intValue14 >>> 11) & mask; + output[28 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; + output[29 + outputOffset] = (intValue15 >>> 13) & mask; + output[30 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; + output[31 + outputOffset] = intValue16 >>> 15; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For27Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For27Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For27Decompress.java (revision 0) @@ -0,0 +1,98 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For27Decompress extends ForDecompress { + static final int numFrameBits = 27; + static final int mask = (int) ((1L<>> 27) | (intValue1 << 5)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 17) | (intValue3 << 15)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 12) | (intValue4 << 20)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 7) | (intValue5 << 25)) & mask; + output[6 + outputOffset] = (intValue5 >>> 2) & mask; + output[7 + outputOffset] = ((intValue5 >>> 29) | (intValue6 << 3)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[9 + outputOffset] = ((intValue7 >>> 19) | (intValue8 << 13)) & mask; + output[10 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; + output[11 + outputOffset] = ((intValue9 >>> 9) | (intValue10 << 23)) & mask; + output[12 + outputOffset] = (intValue10 >>> 4) & mask; + output[13 + outputOffset] = ((intValue10 >>> 31) | (intValue11 << 1)) & mask; + output[14 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[15 + outputOffset] = ((intValue12 >>> 21) | (intValue13 << 11)) & mask; + output[16 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[17 + outputOffset] = ((intValue14 >>> 11) | (intValue15 << 21)) & mask; + output[18 + outputOffset] = ((intValue15 >>> 6) | (intValue16 << 26)) & mask; + output[19 + outputOffset] = (intValue16 >>> 1) & mask; + output[20 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[21 + outputOffset] = ((intValue17 >>> 23) | (intValue18 << 9)) & mask; + output[22 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; + output[23 + outputOffset] = ((intValue19 >>> 13) | (intValue20 << 19)) & mask; + output[24 + outputOffset] = ((intValue20 >>> 8) | (intValue21 << 24)) & mask; + output[25 + outputOffset] = (intValue21 >>> 3) & mask; + output[26 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; + output[27 + outputOffset] = ((intValue22 >>> 25) | (intValue23 << 7)) & mask; + output[28 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; + output[29 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; + output[30 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; + output[31 + outputOffset] = intValue26 >>> 5; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For19Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For19Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For19Decompress.java (revision 0) @@ -0,0 +1,90 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For19Decompress extends ForDecompress { + static final int numFrameBits = 19; + static final int mask = (int) ((1L<>> 19) | (intValue1 << 13)) & mask; + output[2 + outputOffset] = (intValue1 >>> 6) & mask; + output[3 + outputOffset] = ((intValue1 >>> 25) | (intValue2 << 7)) & mask; + output[4 + outputOffset] = (intValue2 >>> 12) & mask; + output[5 + outputOffset] = ((intValue2 >>> 31) | (intValue3 << 1)) & mask; + output[6 + outputOffset] = ((intValue3 >>> 18) | (intValue4 << 14)) & mask; + output[7 + outputOffset] = (intValue4 >>> 5) & mask; + output[8 + outputOffset] = ((intValue4 >>> 24) | (intValue5 << 8)) & mask; + output[9 + outputOffset] = (intValue5 >>> 11) & mask; + output[10 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; + output[11 + outputOffset] = ((intValue6 >>> 17) | (intValue7 << 15)) & mask; + output[12 + outputOffset] = (intValue7 >>> 4) & mask; + output[13 + outputOffset] = ((intValue7 >>> 23) | (intValue8 << 9)) & mask; + output[14 + outputOffset] = (intValue8 >>> 10) & mask; + output[15 + outputOffset] = ((intValue8 >>> 29) | (intValue9 << 3)) & mask; + output[16 + outputOffset] = ((intValue9 >>> 16) | (intValue10 << 16)) & mask; + output[17 + outputOffset] = (intValue10 >>> 3) & mask; + output[18 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; + output[19 + outputOffset] = (intValue11 >>> 9) & mask; + output[20 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; + output[21 + outputOffset] = ((intValue12 >>> 15) | (intValue13 << 17)) & mask; + output[22 + outputOffset] = (intValue13 >>> 2) & mask; + output[23 + outputOffset] = ((intValue13 >>> 21) | (intValue14 << 11)) & mask; + output[24 + outputOffset] = (intValue14 >>> 8) & mask; + output[25 + outputOffset] = ((intValue14 >>> 27) | (intValue15 << 5)) & mask; + output[26 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; + output[27 + outputOffset] = (intValue16 >>> 1) & mask; + output[28 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; + output[29 + outputOffset] = (intValue17 >>> 7) & mask; + output[30 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; + output[31 + outputOffset] = intValue18 >>> 13; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For29Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For29Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For29Decompress.java (revision 0) @@ -0,0 +1,100 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For29Decompress extends ForDecompress { + static final int numFrameBits = 29; + static final int mask = (int) ((1L<>> 29) | (intValue1 << 3)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 26) | (intValue2 << 6)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 23) | (intValue3 << 9)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 20) | (intValue4 << 12)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 17) | (intValue5 << 15)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 14) | (intValue6 << 18)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 11) | (intValue7 << 21)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 8) | (intValue8 << 24)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 5) | (intValue9 << 27)) & mask; + output[10 + outputOffset] = (intValue9 >>> 2) & mask; + output[11 + outputOffset] = ((intValue9 >>> 31) | (intValue10 << 1)) & mask; + output[12 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; + output[13 + outputOffset] = ((intValue11 >>> 25) | (intValue12 << 7)) & mask; + output[14 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; + output[15 + outputOffset] = ((intValue13 >>> 19) | (intValue14 << 13)) & mask; + output[16 + outputOffset] = ((intValue14 >>> 16) | (intValue15 << 16)) & mask; + output[17 + outputOffset] = ((intValue15 >>> 13) | (intValue16 << 19)) & mask; + output[18 + outputOffset] = ((intValue16 >>> 10) | (intValue17 << 22)) & mask; + output[19 + outputOffset] = ((intValue17 >>> 7) | (intValue18 << 25)) & mask; + output[20 + outputOffset] = ((intValue18 >>> 4) | (intValue19 << 28)) & mask; + output[21 + outputOffset] = (intValue19 >>> 1) & mask; + output[22 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; + output[23 + outputOffset] = ((intValue20 >>> 27) | (intValue21 << 5)) & mask; + output[24 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; + output[25 + outputOffset] = ((intValue22 >>> 21) | (intValue23 << 11)) & mask; + output[26 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; + output[27 + outputOffset] = ((intValue24 >>> 15) | (intValue25 << 17)) & mask; + output[28 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; + output[29 + outputOffset] = ((intValue26 >>> 9) | (intValue27 << 23)) & mask; + output[30 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; + output[31 + outputOffset] = intValue28 >>> 3; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For2Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For2Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For2Decompress.java (revision 0) @@ -0,0 +1,73 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For2Decompress extends ForDecompress { + static final int numFrameBits = 2; + static final int mask = (int) ((1L<>> 2) & mask; + output[2 + outputOffset] = (intValue0 >>> 4) & mask; + output[3 + outputOffset] = (intValue0 >>> 6) & mask; + output[4 + outputOffset] = (intValue0 >>> 8) & mask; + output[5 + outputOffset] = (intValue0 >>> 10) & mask; + output[6 + outputOffset] = (intValue0 >>> 12) & mask; + output[7 + outputOffset] = (intValue0 >>> 14) & mask; + output[8 + outputOffset] = (intValue0 >>> 16) & mask; + output[9 + outputOffset] = (intValue0 >>> 18) & mask; + output[10 + outputOffset] = (intValue0 >>> 20) & mask; + output[11 + outputOffset] = (intValue0 >>> 22) & mask; + output[12 + outputOffset] = (intValue0 >>> 24) & mask; + output[13 + outputOffset] = (intValue0 >>> 26) & mask; + output[14 + outputOffset] = (intValue0 >>> 28) & mask; + output[15 + outputOffset] = intValue0 >>> 30; + output[16 + outputOffset] = intValue1 & mask; + output[17 + outputOffset] = (intValue1 >>> 2) & mask; + output[18 + outputOffset] = (intValue1 >>> 4) & mask; + output[19 + outputOffset] = (intValue1 >>> 6) & mask; + output[20 + outputOffset] = (intValue1 >>> 8) & mask; + output[21 + outputOffset] = (intValue1 >>> 10) & mask; + output[22 + outputOffset] = (intValue1 >>> 12) & mask; + output[23 + outputOffset] = (intValue1 >>> 14) & mask; + output[24 + outputOffset] = (intValue1 >>> 16) & mask; + output[25 + outputOffset] = (intValue1 >>> 18) & mask; + output[26 + outputOffset] = (intValue1 >>> 20) & mask; + output[27 + outputOffset] = (intValue1 >>> 22) & mask; + output[28 + outputOffset] = (intValue1 >>> 24) & mask; + output[29 + outputOffset] = (intValue1 >>> 26) & mask; + output[30 + outputOffset] = (intValue1 >>> 28) & mask; + output[31 + outputOffset] = intValue1 >>> 30; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For4Decompress.java (revision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For4Decompress extends ForDecompress { + static final int numFrameBits = 4; + static final int mask = (int) ((1L<>> 4) & mask; + output[2 + outputOffset] = (intValue0 >>> 8) & mask; + output[3 + outputOffset] = (intValue0 >>> 12) & mask; + output[4 + outputOffset] = (intValue0 >>> 16) & mask; + output[5 + outputOffset] = (intValue0 >>> 20) & mask; + output[6 + outputOffset] = (intValue0 >>> 24) & mask; + output[7 + outputOffset] = intValue0 >>> 28; + output[8 + outputOffset] = intValue1 & mask; + output[9 + outputOffset] = (intValue1 >>> 4) & mask; + output[10 + outputOffset] = (intValue1 >>> 8) & mask; + output[11 + outputOffset] = (intValue1 >>> 12) & mask; + output[12 + outputOffset] = (intValue1 >>> 16) & mask; + output[13 + outputOffset] = (intValue1 >>> 20) & mask; + output[14 + outputOffset] = (intValue1 >>> 24) & mask; + output[15 + outputOffset] = intValue1 >>> 28; + output[16 + outputOffset] = intValue2 & mask; + output[17 + outputOffset] = (intValue2 >>> 4) & mask; + output[18 + outputOffset] = (intValue2 >>> 8) & mask; + output[19 + outputOffset] = (intValue2 >>> 12) & mask; + output[20 + outputOffset] = (intValue2 >>> 16) & mask; + output[21 + outputOffset] = (intValue2 >>> 20) & mask; + output[22 + outputOffset] = (intValue2 >>> 24) & mask; + output[23 + outputOffset] = intValue2 >>> 28; + output[24 + outputOffset] = intValue3 & mask; + output[25 + outputOffset] = (intValue3 >>> 4) & mask; + output[26 + outputOffset] = (intValue3 >>> 8) & mask; + output[27 + outputOffset] = (intValue3 >>> 12) & mask; + output[28 + outputOffset] = (intValue3 >>> 16) & mask; + output[29 + outputOffset] = (intValue3 >>> 20) & mask; + output[30 + outputOffset] = (intValue3 >>> 24) & mask; + output[31 + outputOffset] = intValue3 >>> 28; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For6Decompress.java (revision 0) @@ -0,0 +1,77 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For6Decompress extends ForDecompress { + static final int numFrameBits = 6; + static final int mask = (int) ((1L<>> 6) & mask; + output[2 + outputOffset] = (intValue0 >>> 12) & mask; + output[3 + outputOffset] = (intValue0 >>> 18) & mask; + output[4 + outputOffset] = (intValue0 >>> 24) & mask; + output[5 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[6 + outputOffset] = (intValue1 >>> 4) & mask; + output[7 + outputOffset] = (intValue1 >>> 10) & mask; + output[8 + outputOffset] = (intValue1 >>> 16) & mask; + output[9 + outputOffset] = (intValue1 >>> 22) & mask; + output[10 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[11 + outputOffset] = (intValue2 >>> 2) & mask; + output[12 + outputOffset] = (intValue2 >>> 8) & mask; + output[13 + outputOffset] = (intValue2 >>> 14) & mask; + output[14 + outputOffset] = (intValue2 >>> 20) & mask; + output[15 + outputOffset] = intValue2 >>> 26; + output[16 + outputOffset] = intValue3 & mask; + output[17 + outputOffset] = (intValue3 >>> 6) & mask; + output[18 + outputOffset] = (intValue3 >>> 12) & mask; + output[19 + outputOffset] = (intValue3 >>> 18) & mask; + output[20 + outputOffset] = (intValue3 >>> 24) & mask; + output[21 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[22 + outputOffset] = (intValue4 >>> 4) & mask; + output[23 + outputOffset] = (intValue4 >>> 10) & mask; + output[24 + outputOffset] = (intValue4 >>> 16) & mask; + output[25 + outputOffset] = (intValue4 >>> 22) & mask; + output[26 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[27 + outputOffset] = (intValue5 >>> 2) & mask; + output[28 + outputOffset] = (intValue5 >>> 8) & mask; + output[29 + outputOffset] = (intValue5 >>> 14) & mask; + output[30 + outputOffset] = (intValue5 >>> 20) & mask; + output[31 + outputOffset] = intValue5 >>> 26; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For8Decompress.java (revision 0) @@ -0,0 +1,79 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For8Decompress extends ForDecompress { + static final int numFrameBits = 8; + static final int mask = (int) ((1L<>> 8) & mask; + output[2 + outputOffset] = (intValue0 >>> 16) & mask; + output[3 + outputOffset] = intValue0 >>> 24; + output[4 + outputOffset] = intValue1 & mask; + output[5 + outputOffset] = (intValue1 >>> 8) & mask; + output[6 + outputOffset] = (intValue1 >>> 16) & mask; + output[7 + outputOffset] = intValue1 >>> 24; + output[8 + outputOffset] = intValue2 & mask; + output[9 + outputOffset] = (intValue2 >>> 8) & mask; + output[10 + outputOffset] = (intValue2 >>> 16) & mask; + output[11 + outputOffset] = intValue2 >>> 24; + output[12 + outputOffset] = intValue3 & mask; + output[13 + outputOffset] = (intValue3 >>> 8) & mask; + output[14 + outputOffset] = (intValue3 >>> 16) & mask; + output[15 + outputOffset] = intValue3 >>> 24; + output[16 + outputOffset] = intValue4 & mask; + output[17 + outputOffset] = (intValue4 >>> 8) & mask; + output[18 + outputOffset] = (intValue4 >>> 16) & mask; + output[19 + outputOffset] = intValue4 >>> 24; + output[20 + outputOffset] = intValue5 & mask; + output[21 + outputOffset] = (intValue5 >>> 8) & mask; + output[22 + outputOffset] = (intValue5 >>> 16) & mask; + output[23 + outputOffset] = intValue5 >>> 24; + output[24 + outputOffset] = intValue6 & mask; + output[25 + outputOffset] = (intValue6 >>> 8) & mask; + output[26 + outputOffset] = (intValue6 >>> 16) & mask; + output[27 + outputOffset] = intValue6 >>> 24; + output[28 + outputOffset] = intValue7 & mask; + output[29 + outputOffset] = (intValue7 >>> 8) & mask; + output[30 + outputOffset] = (intValue7 >>> 16) & mask; + output[31 + outputOffset] = intValue7 >>> 24; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/gendecompress.py (revision 0) @@ -0,0 +1,121 @@ +""" + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" + +""" +Generate source code for java classes for FOR decompression. +""" + +USE_SCRATCH = False + +def bitsExpr(i, numFrameBits): + framePos = i * numFrameBits + intValNum = (framePos / 32) + bitPos = framePos % 32 + if USE_SCRATCH: + bitsInInt = "inputInts[" + str(intValNum) + "]" + else: + bitsInInt = "intValue" + str(intValNum) + needBrackets = 0 + if bitPos > 0: + bitsInInt += " >>> " + str(bitPos) + needBrackets = 1 + if bitPos + numFrameBits > 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + if USE_SCRATCH: + bitsInInt += " | (inputInts[" + str(intValNum+1) + "] << "+ str(32 - bitPos) + ")" + else: + bitsInInt += " | (intValue" + str(intValNum+1) + " << "+ str(32 - bitPos) + ")" + needBrackets = 1 + if bitPos + numFrameBits != 32: + if needBrackets: + bitsInInt = "(" + bitsInInt + ")" + bitsInInt += " & mask" + return bitsInInt + + +def genDecompressClass(numFrameBits): + className = "For" + str(numFrameBits) + "Decompress" + fileName = className + ".java" + imports = "import java.nio.IntBuffer;\n" + f = open(fileName, 'w') + w = f.write + try: + w("package org.apache.lucene.util.pfor;\n") + w("""/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */""") + w("\n/* This program is generated, do not modify. See gendecompress.py */\n\n") + w("import java.nio.IntBuffer;\n") + w("class " + className + " extends ForDecompress {\n") + w(" static final int numFrameBits = " + str(numFrameBits) + ";\n") + w(" static final int mask = (int) ((1L<= 32) {\n") + w(' for(int step=0;step<4;step++) {\n') + if USE_SCRATCH: + w(' compressedBuffer.get(inputInts, 0, %d);\n' % numFrameBits) + else: + for i in range(numFrameBits): # declare int vars and init from buffer + w(" int intValue" + str(i) + " = compressedBuffer.get();\n") + + for i in range(32): # set output from int vars + w(" output[" + str(i) + " + outputOffset] = " + bitsExpr(i, numFrameBits) + ";\n") + w(""" // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} +""") + finally: f.close() + + + +def genDecompressClasses(): + numFrameBits = 1 + while numFrameBits <= 31: # 32 special case, not generated. + genDecompressClass(numFrameBits) + numFrameBits += 1 + + + +if __name__ == "__main__": + genDecompressClasses() Index: lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For10Decompress.java (revision 0) @@ -0,0 +1,81 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For10Decompress extends ForDecompress { + static final int numFrameBits = 10; + static final int mask = (int) ((1L<>> 10) & mask; + output[2 + outputOffset] = (intValue0 >>> 20) & mask; + output[3 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[4 + outputOffset] = (intValue1 >>> 8) & mask; + output[5 + outputOffset] = (intValue1 >>> 18) & mask; + output[6 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[7 + outputOffset] = (intValue2 >>> 6) & mask; + output[8 + outputOffset] = (intValue2 >>> 16) & mask; + output[9 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[10 + outputOffset] = (intValue3 >>> 4) & mask; + output[11 + outputOffset] = (intValue3 >>> 14) & mask; + output[12 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[13 + outputOffset] = (intValue4 >>> 2) & mask; + output[14 + outputOffset] = (intValue4 >>> 12) & mask; + output[15 + outputOffset] = intValue4 >>> 22; + output[16 + outputOffset] = intValue5 & mask; + output[17 + outputOffset] = (intValue5 >>> 10) & mask; + output[18 + outputOffset] = (intValue5 >>> 20) & mask; + output[19 + outputOffset] = ((intValue5 >>> 30) | (intValue6 << 2)) & mask; + output[20 + outputOffset] = (intValue6 >>> 8) & mask; + output[21 + outputOffset] = (intValue6 >>> 18) & mask; + output[22 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[23 + outputOffset] = (intValue7 >>> 6) & mask; + output[24 + outputOffset] = (intValue7 >>> 16) & mask; + output[25 + outputOffset] = ((intValue7 >>> 26) | (intValue8 << 6)) & mask; + output[26 + outputOffset] = (intValue8 >>> 4) & mask; + output[27 + outputOffset] = (intValue8 >>> 14) & mask; + output[28 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[29 + outputOffset] = (intValue9 >>> 2) & mask; + output[30 + outputOffset] = (intValue9 >>> 12) & mask; + output[31 + outputOffset] = intValue9 >>> 22; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For20Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For20Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For20Decompress.java (revision 0) @@ -0,0 +1,91 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For20Decompress extends ForDecompress { + static final int numFrameBits = 20; + static final int mask = (int) ((1L<>> 20) | (intValue1 << 12)) & mask; + output[2 + outputOffset] = (intValue1 >>> 8) & mask; + output[3 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[4 + outputOffset] = ((intValue2 >>> 16) | (intValue3 << 16)) & mask; + output[5 + outputOffset] = (intValue3 >>> 4) & mask; + output[6 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[7 + outputOffset] = intValue4 >>> 12; + output[8 + outputOffset] = intValue5 & mask; + output[9 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[10 + outputOffset] = (intValue6 >>> 8) & mask; + output[11 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[12 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[13 + outputOffset] = (intValue8 >>> 4) & mask; + output[14 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[15 + outputOffset] = intValue9 >>> 12; + output[16 + outputOffset] = intValue10 & mask; + output[17 + outputOffset] = ((intValue10 >>> 20) | (intValue11 << 12)) & mask; + output[18 + outputOffset] = (intValue11 >>> 8) & mask; + output[19 + outputOffset] = ((intValue11 >>> 28) | (intValue12 << 4)) & mask; + output[20 + outputOffset] = ((intValue12 >>> 16) | (intValue13 << 16)) & mask; + output[21 + outputOffset] = (intValue13 >>> 4) & mask; + output[22 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; + output[23 + outputOffset] = intValue14 >>> 12; + output[24 + outputOffset] = intValue15 & mask; + output[25 + outputOffset] = ((intValue15 >>> 20) | (intValue16 << 12)) & mask; + output[26 + outputOffset] = (intValue16 >>> 8) & mask; + output[27 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[28 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; + output[29 + outputOffset] = (intValue18 >>> 4) & mask; + output[30 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[31 + outputOffset] = intValue19 >>> 12; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/package.html =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/package.html (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/package.html (revision 0) @@ -0,0 +1,25 @@ + + + + + + + +Classes dealing with (patched) frame of reference compression and decompression. + + Index: lucene/src/java/org/apache/lucene/util/pfor/For30Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For30Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For30Decompress.java (revision 0) @@ -0,0 +1,101 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For30Decompress extends ForDecompress { + static final int numFrameBits = 30; + static final int mask = (int) ((1L<>> 30) | (intValue1 << 2)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 22) | (intValue5 << 10)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[7 + outputOffset] = ((intValue6 >>> 18) | (intValue7 << 14)) & mask; + output[8 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[9 + outputOffset] = ((intValue8 >>> 14) | (intValue9 << 18)) & mask; + output[10 + outputOffset] = ((intValue9 >>> 12) | (intValue10 << 20)) & mask; + output[11 + outputOffset] = ((intValue10 >>> 10) | (intValue11 << 22)) & mask; + output[12 + outputOffset] = ((intValue11 >>> 8) | (intValue12 << 24)) & mask; + output[13 + outputOffset] = ((intValue12 >>> 6) | (intValue13 << 26)) & mask; + output[14 + outputOffset] = ((intValue13 >>> 4) | (intValue14 << 28)) & mask; + output[15 + outputOffset] = intValue14 >>> 2; + output[16 + outputOffset] = intValue15 & mask; + output[17 + outputOffset] = ((intValue15 >>> 30) | (intValue16 << 2)) & mask; + output[18 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[19 + outputOffset] = ((intValue17 >>> 26) | (intValue18 << 6)) & mask; + output[20 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[21 + outputOffset] = ((intValue19 >>> 22) | (intValue20 << 10)) & mask; + output[22 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; + output[23 + outputOffset] = ((intValue21 >>> 18) | (intValue22 << 14)) & mask; + output[24 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; + output[25 + outputOffset] = ((intValue23 >>> 14) | (intValue24 << 18)) & mask; + output[26 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; + output[27 + outputOffset] = ((intValue25 >>> 10) | (intValue26 << 22)) & mask; + output[28 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; + output[29 + outputOffset] = ((intValue27 >>> 6) | (intValue28 << 26)) & mask; + output[30 + outputOffset] = ((intValue28 >>> 4) | (intValue29 << 28)) & mask; + output[31 + outputOffset] = intValue29 >>> 2; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For12Decompress.java (revision 0) @@ -0,0 +1,83 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For12Decompress extends ForDecompress { + static final int numFrameBits = 12; + static final int mask = (int) ((1L<>> 12) & mask; + output[2 + outputOffset] = ((intValue0 >>> 24) | (intValue1 << 8)) & mask; + output[3 + outputOffset] = (intValue1 >>> 4) & mask; + output[4 + outputOffset] = (intValue1 >>> 16) & mask; + output[5 + outputOffset] = ((intValue1 >>> 28) | (intValue2 << 4)) & mask; + output[6 + outputOffset] = (intValue2 >>> 8) & mask; + output[7 + outputOffset] = intValue2 >>> 20; + output[8 + outputOffset] = intValue3 & mask; + output[9 + outputOffset] = (intValue3 >>> 12) & mask; + output[10 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[11 + outputOffset] = (intValue4 >>> 4) & mask; + output[12 + outputOffset] = (intValue4 >>> 16) & mask; + output[13 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[14 + outputOffset] = (intValue5 >>> 8) & mask; + output[15 + outputOffset] = intValue5 >>> 20; + output[16 + outputOffset] = intValue6 & mask; + output[17 + outputOffset] = (intValue6 >>> 12) & mask; + output[18 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[19 + outputOffset] = (intValue7 >>> 4) & mask; + output[20 + outputOffset] = (intValue7 >>> 16) & mask; + output[21 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[22 + outputOffset] = (intValue8 >>> 8) & mask; + output[23 + outputOffset] = intValue8 >>> 20; + output[24 + outputOffset] = intValue9 & mask; + output[25 + outputOffset] = (intValue9 >>> 12) & mask; + output[26 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[27 + outputOffset] = (intValue10 >>> 4) & mask; + output[28 + outputOffset] = (intValue10 >>> 16) & mask; + output[29 + outputOffset] = ((intValue10 >>> 28) | (intValue11 << 4)) & mask; + output[30 + outputOffset] = (intValue11 >>> 8) & mask; + output[31 + outputOffset] = intValue11 >>> 20; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For22Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For22Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For22Decompress.java (revision 0) @@ -0,0 +1,93 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For22Decompress extends ForDecompress { + static final int numFrameBits = 22; + static final int mask = (int) ((1L<>> 22) | (intValue1 << 10)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 12) | (intValue2 << 20)) & mask; + output[3 + outputOffset] = (intValue2 >>> 2) & mask; + output[4 + outputOffset] = ((intValue2 >>> 24) | (intValue3 << 8)) & mask; + output[5 + outputOffset] = ((intValue3 >>> 14) | (intValue4 << 18)) & mask; + output[6 + outputOffset] = (intValue4 >>> 4) & mask; + output[7 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[8 + outputOffset] = ((intValue5 >>> 16) | (intValue6 << 16)) & mask; + output[9 + outputOffset] = (intValue6 >>> 6) & mask; + output[10 + outputOffset] = ((intValue6 >>> 28) | (intValue7 << 4)) & mask; + output[11 + outputOffset] = ((intValue7 >>> 18) | (intValue8 << 14)) & mask; + output[12 + outputOffset] = (intValue8 >>> 8) & mask; + output[13 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[14 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[15 + outputOffset] = intValue10 >>> 10; + output[16 + outputOffset] = intValue11 & mask; + output[17 + outputOffset] = ((intValue11 >>> 22) | (intValue12 << 10)) & mask; + output[18 + outputOffset] = ((intValue12 >>> 12) | (intValue13 << 20)) & mask; + output[19 + outputOffset] = (intValue13 >>> 2) & mask; + output[20 + outputOffset] = ((intValue13 >>> 24) | (intValue14 << 8)) & mask; + output[21 + outputOffset] = ((intValue14 >>> 14) | (intValue15 << 18)) & mask; + output[22 + outputOffset] = (intValue15 >>> 4) & mask; + output[23 + outputOffset] = ((intValue15 >>> 26) | (intValue16 << 6)) & mask; + output[24 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; + output[25 + outputOffset] = (intValue17 >>> 6) & mask; + output[26 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; + output[27 + outputOffset] = ((intValue18 >>> 18) | (intValue19 << 14)) & mask; + output[28 + outputOffset] = (intValue19 >>> 8) & mask; + output[29 + outputOffset] = ((intValue19 >>> 30) | (intValue20 << 2)) & mask; + output[30 + outputOffset] = ((intValue20 >>> 20) | (intValue21 << 12)) & mask; + output[31 + outputOffset] = intValue21 >>> 10; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For32Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For32Decompress.java (revision 0) @@ -0,0 +1,27 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.IntBuffer; +class For32Decompress extends ForDecompress { + static void decompressFrame(FrameOfRef frameOfRef) { + int oldBufPos = frameOfRef.compressedBuffer.position(); + frameOfRef.compressedBuffer.position(frameOfRef.COMPRESSED_INDEX); + frameOfRef.compressedBuffer.get(frameOfRef.unCompressedData, frameOfRef.offset, frameOfRef.unComprSize); + frameOfRef.compressedBuffer.position(oldBufPos); + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For14Decompress.java (revision 0) @@ -0,0 +1,85 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For14Decompress extends ForDecompress { + static final int numFrameBits = 14; + static final int mask = (int) ((1L<>> 14) & mask; + output[2 + outputOffset] = ((intValue0 >>> 28) | (intValue1 << 4)) & mask; + output[3 + outputOffset] = (intValue1 >>> 10) & mask; + output[4 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; + output[5 + outputOffset] = (intValue2 >>> 6) & mask; + output[6 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[7 + outputOffset] = (intValue3 >>> 2) & mask; + output[8 + outputOffset] = (intValue3 >>> 16) & mask; + output[9 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[10 + outputOffset] = (intValue4 >>> 12) & mask; + output[11 + outputOffset] = ((intValue4 >>> 26) | (intValue5 << 6)) & mask; + output[12 + outputOffset] = (intValue5 >>> 8) & mask; + output[13 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[14 + outputOffset] = (intValue6 >>> 4) & mask; + output[15 + outputOffset] = intValue6 >>> 18; + output[16 + outputOffset] = intValue7 & mask; + output[17 + outputOffset] = (intValue7 >>> 14) & mask; + output[18 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[19 + outputOffset] = (intValue8 >>> 10) & mask; + output[20 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[21 + outputOffset] = (intValue9 >>> 6) & mask; + output[22 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[23 + outputOffset] = (intValue10 >>> 2) & mask; + output[24 + outputOffset] = (intValue10 >>> 16) & mask; + output[25 + outputOffset] = ((intValue10 >>> 30) | (intValue11 << 2)) & mask; + output[26 + outputOffset] = (intValue11 >>> 12) & mask; + output[27 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[28 + outputOffset] = (intValue12 >>> 8) & mask; + output[29 + outputOffset] = ((intValue12 >>> 22) | (intValue13 << 10)) & mask; + output[30 + outputOffset] = (intValue13 >>> 4) & mask; + output[31 + outputOffset] = intValue13 >>> 18; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For24Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For24Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For24Decompress.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For24Decompress extends ForDecompress { + static final int numFrameBits = 24; + static final int mask = (int) ((1L<>> 24) | (intValue1 << 8)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 16) | (intValue2 << 16)) & mask; + output[3 + outputOffset] = intValue2 >>> 8; + output[4 + outputOffset] = intValue3 & mask; + output[5 + outputOffset] = ((intValue3 >>> 24) | (intValue4 << 8)) & mask; + output[6 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; + output[7 + outputOffset] = intValue5 >>> 8; + output[8 + outputOffset] = intValue6 & mask; + output[9 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[10 + outputOffset] = ((intValue7 >>> 16) | (intValue8 << 16)) & mask; + output[11 + outputOffset] = intValue8 >>> 8; + output[12 + outputOffset] = intValue9 & mask; + output[13 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[14 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[15 + outputOffset] = intValue11 >>> 8; + output[16 + outputOffset] = intValue12 & mask; + output[17 + outputOffset] = ((intValue12 >>> 24) | (intValue13 << 8)) & mask; + output[18 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[19 + outputOffset] = intValue14 >>> 8; + output[20 + outputOffset] = intValue15 & mask; + output[21 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[22 + outputOffset] = ((intValue16 >>> 16) | (intValue17 << 16)) & mask; + output[23 + outputOffset] = intValue17 >>> 8; + output[24 + outputOffset] = intValue18 & mask; + output[25 + outputOffset] = ((intValue18 >>> 24) | (intValue19 << 8)) & mask; + output[26 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; + output[27 + outputOffset] = intValue20 >>> 8; + output[28 + outputOffset] = intValue21 & mask; + output[29 + outputOffset] = ((intValue21 >>> 24) | (intValue22 << 8)) & mask; + output[30 + outputOffset] = ((intValue22 >>> 16) | (intValue23 << 16)) & mask; + output[31 + outputOffset] = intValue23 >>> 8; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For16Decompress.java (revision 0) @@ -0,0 +1,87 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For16Decompress extends ForDecompress { + static final int numFrameBits = 16; + static final int mask = (int) ((1L<>> 16; + output[2 + outputOffset] = intValue1 & mask; + output[3 + outputOffset] = intValue1 >>> 16; + output[4 + outputOffset] = intValue2 & mask; + output[5 + outputOffset] = intValue2 >>> 16; + output[6 + outputOffset] = intValue3 & mask; + output[7 + outputOffset] = intValue3 >>> 16; + output[8 + outputOffset] = intValue4 & mask; + output[9 + outputOffset] = intValue4 >>> 16; + output[10 + outputOffset] = intValue5 & mask; + output[11 + outputOffset] = intValue5 >>> 16; + output[12 + outputOffset] = intValue6 & mask; + output[13 + outputOffset] = intValue6 >>> 16; + output[14 + outputOffset] = intValue7 & mask; + output[15 + outputOffset] = intValue7 >>> 16; + output[16 + outputOffset] = intValue8 & mask; + output[17 + outputOffset] = intValue8 >>> 16; + output[18 + outputOffset] = intValue9 & mask; + output[19 + outputOffset] = intValue9 >>> 16; + output[20 + outputOffset] = intValue10 & mask; + output[21 + outputOffset] = intValue10 >>> 16; + output[22 + outputOffset] = intValue11 & mask; + output[23 + outputOffset] = intValue11 >>> 16; + output[24 + outputOffset] = intValue12 & mask; + output[25 + outputOffset] = intValue12 >>> 16; + output[26 + outputOffset] = intValue13 & mask; + output[27 + outputOffset] = intValue13 >>> 16; + output[28 + outputOffset] = intValue14 & mask; + output[29 + outputOffset] = intValue14 >>> 16; + output[30 + outputOffset] = intValue15 & mask; + output[31 + outputOffset] = intValue15 >>> 16; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For26Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For26Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For26Decompress.java (revision 0) @@ -0,0 +1,97 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For26Decompress extends ForDecompress { + static final int numFrameBits = 26; + static final int mask = (int) ((1L<>> 26) | (intValue1 << 6)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 20) | (intValue2 << 12)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 14) | (intValue3 << 18)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 8) | (intValue4 << 24)) & mask; + output[5 + outputOffset] = (intValue4 >>> 2) & mask; + output[6 + outputOffset] = ((intValue4 >>> 28) | (intValue5 << 4)) & mask; + output[7 + outputOffset] = ((intValue5 >>> 22) | (intValue6 << 10)) & mask; + output[8 + outputOffset] = ((intValue6 >>> 16) | (intValue7 << 16)) & mask; + output[9 + outputOffset] = ((intValue7 >>> 10) | (intValue8 << 22)) & mask; + output[10 + outputOffset] = (intValue8 >>> 4) & mask; + output[11 + outputOffset] = ((intValue8 >>> 30) | (intValue9 << 2)) & mask; + output[12 + outputOffset] = ((intValue9 >>> 24) | (intValue10 << 8)) & mask; + output[13 + outputOffset] = ((intValue10 >>> 18) | (intValue11 << 14)) & mask; + output[14 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; + output[15 + outputOffset] = intValue12 >>> 6; + output[16 + outputOffset] = intValue13 & mask; + output[17 + outputOffset] = ((intValue13 >>> 26) | (intValue14 << 6)) & mask; + output[18 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; + output[19 + outputOffset] = ((intValue15 >>> 14) | (intValue16 << 18)) & mask; + output[20 + outputOffset] = ((intValue16 >>> 8) | (intValue17 << 24)) & mask; + output[21 + outputOffset] = (intValue17 >>> 2) & mask; + output[22 + outputOffset] = ((intValue17 >>> 28) | (intValue18 << 4)) & mask; + output[23 + outputOffset] = ((intValue18 >>> 22) | (intValue19 << 10)) & mask; + output[24 + outputOffset] = ((intValue19 >>> 16) | (intValue20 << 16)) & mask; + output[25 + outputOffset] = ((intValue20 >>> 10) | (intValue21 << 22)) & mask; + output[26 + outputOffset] = (intValue21 >>> 4) & mask; + output[27 + outputOffset] = ((intValue21 >>> 30) | (intValue22 << 2)) & mask; + output[28 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; + output[29 + outputOffset] = ((intValue23 >>> 18) | (intValue24 << 14)) & mask; + output[30 + outputOffset] = ((intValue24 >>> 12) | (intValue25 << 20)) & mask; + output[31 + outputOffset] = intValue25 >>> 6; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For18Decompress.java (revision 0) @@ -0,0 +1,89 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For18Decompress extends ForDecompress { + static final int numFrameBits = 18; + static final int mask = (int) ((1L<>> 18) | (intValue1 << 14)) & mask; + output[2 + outputOffset] = (intValue1 >>> 4) & mask; + output[3 + outputOffset] = ((intValue1 >>> 22) | (intValue2 << 10)) & mask; + output[4 + outputOffset] = (intValue2 >>> 8) & mask; + output[5 + outputOffset] = ((intValue2 >>> 26) | (intValue3 << 6)) & mask; + output[6 + outputOffset] = (intValue3 >>> 12) & mask; + output[7 + outputOffset] = ((intValue3 >>> 30) | (intValue4 << 2)) & mask; + output[8 + outputOffset] = ((intValue4 >>> 16) | (intValue5 << 16)) & mask; + output[9 + outputOffset] = (intValue5 >>> 2) & mask; + output[10 + outputOffset] = ((intValue5 >>> 20) | (intValue6 << 12)) & mask; + output[11 + outputOffset] = (intValue6 >>> 6) & mask; + output[12 + outputOffset] = ((intValue6 >>> 24) | (intValue7 << 8)) & mask; + output[13 + outputOffset] = (intValue7 >>> 10) & mask; + output[14 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[15 + outputOffset] = intValue8 >>> 14; + output[16 + outputOffset] = intValue9 & mask; + output[17 + outputOffset] = ((intValue9 >>> 18) | (intValue10 << 14)) & mask; + output[18 + outputOffset] = (intValue10 >>> 4) & mask; + output[19 + outputOffset] = ((intValue10 >>> 22) | (intValue11 << 10)) & mask; + output[20 + outputOffset] = (intValue11 >>> 8) & mask; + output[21 + outputOffset] = ((intValue11 >>> 26) | (intValue12 << 6)) & mask; + output[22 + outputOffset] = (intValue12 >>> 12) & mask; + output[23 + outputOffset] = ((intValue12 >>> 30) | (intValue13 << 2)) & mask; + output[24 + outputOffset] = ((intValue13 >>> 16) | (intValue14 << 16)) & mask; + output[25 + outputOffset] = (intValue14 >>> 2) & mask; + output[26 + outputOffset] = ((intValue14 >>> 20) | (intValue15 << 12)) & mask; + output[27 + outputOffset] = (intValue15 >>> 6) & mask; + output[28 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[29 + outputOffset] = (intValue16 >>> 10) & mask; + output[30 + outputOffset] = ((intValue16 >>> 28) | (intValue17 << 4)) & mask; + output[31 + outputOffset] = intValue17 >>> 14; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For28Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For28Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For28Decompress.java (revision 0) @@ -0,0 +1,99 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For28Decompress extends ForDecompress { + static final int numFrameBits = 28; + static final int mask = (int) ((1L<>> 28) | (intValue1 << 4)) & mask; + output[2 + outputOffset] = ((intValue1 >>> 24) | (intValue2 << 8)) & mask; + output[3 + outputOffset] = ((intValue2 >>> 20) | (intValue3 << 12)) & mask; + output[4 + outputOffset] = ((intValue3 >>> 16) | (intValue4 << 16)) & mask; + output[5 + outputOffset] = ((intValue4 >>> 12) | (intValue5 << 20)) & mask; + output[6 + outputOffset] = ((intValue5 >>> 8) | (intValue6 << 24)) & mask; + output[7 + outputOffset] = intValue6 >>> 4; + output[8 + outputOffset] = intValue7 & mask; + output[9 + outputOffset] = ((intValue7 >>> 28) | (intValue8 << 4)) & mask; + output[10 + outputOffset] = ((intValue8 >>> 24) | (intValue9 << 8)) & mask; + output[11 + outputOffset] = ((intValue9 >>> 20) | (intValue10 << 12)) & mask; + output[12 + outputOffset] = ((intValue10 >>> 16) | (intValue11 << 16)) & mask; + output[13 + outputOffset] = ((intValue11 >>> 12) | (intValue12 << 20)) & mask; + output[14 + outputOffset] = ((intValue12 >>> 8) | (intValue13 << 24)) & mask; + output[15 + outputOffset] = intValue13 >>> 4; + output[16 + outputOffset] = intValue14 & mask; + output[17 + outputOffset] = ((intValue14 >>> 28) | (intValue15 << 4)) & mask; + output[18 + outputOffset] = ((intValue15 >>> 24) | (intValue16 << 8)) & mask; + output[19 + outputOffset] = ((intValue16 >>> 20) | (intValue17 << 12)) & mask; + output[20 + outputOffset] = ((intValue17 >>> 16) | (intValue18 << 16)) & mask; + output[21 + outputOffset] = ((intValue18 >>> 12) | (intValue19 << 20)) & mask; + output[22 + outputOffset] = ((intValue19 >>> 8) | (intValue20 << 24)) & mask; + output[23 + outputOffset] = intValue20 >>> 4; + output[24 + outputOffset] = intValue21 & mask; + output[25 + outputOffset] = ((intValue21 >>> 28) | (intValue22 << 4)) & mask; + output[26 + outputOffset] = ((intValue22 >>> 24) | (intValue23 << 8)) & mask; + output[27 + outputOffset] = ((intValue23 >>> 20) | (intValue24 << 12)) & mask; + output[28 + outputOffset] = ((intValue24 >>> 16) | (intValue25 << 16)) & mask; + output[29 + outputOffset] = ((intValue25 >>> 12) | (intValue26 << 20)) & mask; + output[30 + outputOffset] = ((intValue26 >>> 8) | (intValue27 << 24)) & mask; + output[31 + outputOffset] = intValue27 >>> 4; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For1Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For1Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For1Decompress.java (revision 0) @@ -0,0 +1,72 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For1Decompress extends ForDecompress { + static final int numFrameBits = 1; + static final int mask = (int) ((1L<>> 1) & mask; + output[2 + outputOffset] = (intValue0 >>> 2) & mask; + output[3 + outputOffset] = (intValue0 >>> 3) & mask; + output[4 + outputOffset] = (intValue0 >>> 4) & mask; + output[5 + outputOffset] = (intValue0 >>> 5) & mask; + output[6 + outputOffset] = (intValue0 >>> 6) & mask; + output[7 + outputOffset] = (intValue0 >>> 7) & mask; + output[8 + outputOffset] = (intValue0 >>> 8) & mask; + output[9 + outputOffset] = (intValue0 >>> 9) & mask; + output[10 + outputOffset] = (intValue0 >>> 10) & mask; + output[11 + outputOffset] = (intValue0 >>> 11) & mask; + output[12 + outputOffset] = (intValue0 >>> 12) & mask; + output[13 + outputOffset] = (intValue0 >>> 13) & mask; + output[14 + outputOffset] = (intValue0 >>> 14) & mask; + output[15 + outputOffset] = (intValue0 >>> 15) & mask; + output[16 + outputOffset] = (intValue0 >>> 16) & mask; + output[17 + outputOffset] = (intValue0 >>> 17) & mask; + output[18 + outputOffset] = (intValue0 >>> 18) & mask; + output[19 + outputOffset] = (intValue0 >>> 19) & mask; + output[20 + outputOffset] = (intValue0 >>> 20) & mask; + output[21 + outputOffset] = (intValue0 >>> 21) & mask; + output[22 + outputOffset] = (intValue0 >>> 22) & mask; + output[23 + outputOffset] = (intValue0 >>> 23) & mask; + output[24 + outputOffset] = (intValue0 >>> 24) & mask; + output[25 + outputOffset] = (intValue0 >>> 25) & mask; + output[26 + outputOffset] = (intValue0 >>> 26) & mask; + output[27 + outputOffset] = (intValue0 >>> 27) & mask; + output[28 + outputOffset] = (intValue0 >>> 28) & mask; + output[29 + outputOffset] = (intValue0 >>> 29) & mask; + output[30 + outputOffset] = (intValue0 >>> 30) & mask; + output[31 + outputOffset] = intValue0 >>> 31; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +} Index: lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java =================================================================== --- lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java (revision 0) +++ lucene/src/java/org/apache/lucene/util/pfor/For3Decompress.java (revision 0) @@ -0,0 +1,74 @@ +package org.apache.lucene.util.pfor; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/* This program is generated, do not modify. See gendecompress.py */ + +import java.nio.IntBuffer; +class For3Decompress extends ForDecompress { + static final int numFrameBits = 3; + static final int mask = (int) ((1L<>> 3) & mask; + output[2 + outputOffset] = (intValue0 >>> 6) & mask; + output[3 + outputOffset] = (intValue0 >>> 9) & mask; + output[4 + outputOffset] = (intValue0 >>> 12) & mask; + output[5 + outputOffset] = (intValue0 >>> 15) & mask; + output[6 + outputOffset] = (intValue0 >>> 18) & mask; + output[7 + outputOffset] = (intValue0 >>> 21) & mask; + output[8 + outputOffset] = (intValue0 >>> 24) & mask; + output[9 + outputOffset] = (intValue0 >>> 27) & mask; + output[10 + outputOffset] = ((intValue0 >>> 30) | (intValue1 << 2)) & mask; + output[11 + outputOffset] = (intValue1 >>> 1) & mask; + output[12 + outputOffset] = (intValue1 >>> 4) & mask; + output[13 + outputOffset] = (intValue1 >>> 7) & mask; + output[14 + outputOffset] = (intValue1 >>> 10) & mask; + output[15 + outputOffset] = (intValue1 >>> 13) & mask; + output[16 + outputOffset] = (intValue1 >>> 16) & mask; + output[17 + outputOffset] = (intValue1 >>> 19) & mask; + output[18 + outputOffset] = (intValue1 >>> 22) & mask; + output[19 + outputOffset] = (intValue1 >>> 25) & mask; + output[20 + outputOffset] = (intValue1 >>> 28) & mask; + output[21 + outputOffset] = ((intValue1 >>> 31) | (intValue2 << 1)) & mask; + output[22 + outputOffset] = (intValue2 >>> 2) & mask; + output[23 + outputOffset] = (intValue2 >>> 5) & mask; + output[24 + outputOffset] = (intValue2 >>> 8) & mask; + output[25 + outputOffset] = (intValue2 >>> 11) & mask; + output[26 + outputOffset] = (intValue2 >>> 14) & mask; + output[27 + outputOffset] = (intValue2 >>> 17) & mask; + output[28 + outputOffset] = (intValue2 >>> 20) & mask; + output[29 + outputOffset] = (intValue2 >>> 23) & mask; + output[30 + outputOffset] = (intValue2 >>> 26) & mask; + output[31 + outputOffset] = intValue2 >>> 29; + // inputSize -= 32; + outputOffset += 32; + } + + //if (inputSize > 0) { + // decodeAnyFrame(compressedBuffer, bufIndex, inputSize, numFrameBits, output, outputOffset); + //} + } +}