Index: build.xml =================================================================== --- build.xml (revision 1367066) +++ build.xml (working copy) @@ -24,7 +24,7 @@ - + @@ -65,7 +65,7 @@ - + @@ -74,13 +74,19 @@ - + + + + + + + @@ -116,7 +122,7 @@ - + @@ -129,7 +135,7 @@ - + @@ -138,7 +144,6 @@ File | Project Structure | Project | Project SDK. - @@ -176,7 +181,7 @@ - + Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (revision 1367066) +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -283,8 +283,7 @@ "MockVariableIntBlock", "MockSep", "MockRandom", - "For", - "PFor" + "For" )); // ----------------------------------------------------------------- Index: lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (revision 1367066) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (working copy) @@ -61,7 +61,6 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.apache.lucene.codecs.pfor.*; /** * Randomly combines terms index impl w/ postings impls. @@ -103,8 +102,6 @@ final int baseBlockSize = _TestUtil.nextInt(random, 1, 127); delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize)); // TODO: others - delegates.add(new ForFactory()); - delegates.add(new PForFactory()); } private static String getExtension(String fileName) { Index: lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat =================================================================== --- lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (revision 1367066) +++ lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (working copy) @@ -17,8 +17,6 @@ org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat org.apache.lucene.codecs.memory.MemoryPostingsFormat -org.apache.lucene.codecs.pfor.ForPostingsFormat -org.apache.lucene.codecs.pfor.PForPostingsFormat org.apache.lucene.codecs.bulkvint.BulkVIntPostingsFormat org.apache.lucene.codecs.block.BlockPostingsFormat org.apache.lucene.codecs.memory.DirectPostingsFormat Index: lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java (revision 1367066) +++ lucene/core/src/test/org/apache/lucene/codecs/pfor/TestPForUtil.java (working copy) @@ -1,293 +0,0 @@ -package org.apache.lucene.codecs.pfor; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.nio.ByteBuffer; -import java.nio.IntBuffer; -import java.util.Arrays; -import java.util.Collections; -import java.util.Locale; -import java.util.Random; - -import org.apache.lucene.codecs.pfor.ForPostingsFormat; -import org.apache.lucene.codecs.pfor.PForUtil; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Test the core utility for PFor compress and decompress - * We don't specially provide test case for For encoder/decoder, since - * PFor is a extended version of For, and most methods will be reused - * here. - */ -public class TestPForUtil extends LuceneTestCase { - static final int[] MASK={ 0x00000000, - 0x00000001, 0x00000003, 0x00000007, 0x0000000f, 0x0000001f, 0x0000003f, - 0x0000007f, 0x000000ff, 0x000001ff, 0x000003ff, 0x000007ff, 0x00000fff, - 0x00001fff, 0x00003fff, 0x00007fff, 0x0000ffff, 0x0001ffff, 0x0003ffff, - 0x0007ffff, 0x000fffff, 0x001fffff, 0x003fffff, 0x007fffff, 0x00ffffff, - 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff, - 0x7fffffff, 0xffffffff}; - Random gen; - public void initRandom() { - this.gen = random(); - } - - /** - * Should not encode extra information other than single int - */ - public void testAllEqual() throws Exception { - initRandom(); - int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE; - int[] data=new int[sz]; - byte[] res = new byte[sz*8]; - int[] copy = new int[sz]; - IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer(); - int ensz; - int header; - - Arrays.fill(data,gen.nextInt()); - header = ForUtil.compress(data,resBuffer); // test For - ensz = ForUtil.getEncodedSize(header); - assert ensz == 4; - - ForUtil.decompress(resBuffer,copy,header); - assert cmp(data,sz,copy,sz)==true; - - Arrays.fill(data,gen.nextInt()); - header = PForUtil.compress(data,resBuffer); // test PFor - ensz = PForUtil.getEncodedSize(header); - assert ensz == 4; - - PForUtil.decompress(resBuffer,copy,header); - assert cmp(data,sz,copy,sz)==true; - } - - /** - * Test correctness of forced exception. - * the forced ones should exactly fit max chain - */ - public void testForcedExceptionDistance() throws Exception { - initRandom(); - int sz=ForPostingsFormat.DEFAULT_BLOCK_SIZE; - int[] data=new int[sz]; - byte[] res = new byte[sz*8]; - int[] copy = new int[sz]; - IntBuffer resBuffer = ByteBuffer.wrap(res).asIntBuffer(); - int numBits = gen.nextInt(5)+1; - - int i,j; - int pace, ensz, header; - int expect, got; - - // fill exception value with same pace, there should - // be no forced exceptions. - createDistribution(data, sz, 1, MASK[numBits], MASK[numBits]); - pace = 1<0 - createDistribution(data, sz, 1, MASK[numBits], MASK[numBits]); - pace = (1< "+sz*8; // must not exceed the loose upperbound - assert (ensz >= 4); // at least we have an exception, right? - - PForUtil.decompress(resBuffer,copy,header); - -// println(getHex(data,sz)+"\n"); -// println(getHex(res,ensz)+"\n"); -// println(getHex(copy,sz)+"\n"); - - // fetch the last int, i.e. last exception. - int lastExc = (res[ensz-4] << 24) | - ((0xff & res[ensz-3]) << 16) | - ((0xff & res[ensz-2]) << 8 ) | - (0xff & res[ensz-1]); - - // trailing forced exceptions are suppressed, - // so the last exception should be what we assigned. - assert lastExc==excValue; - assert cmp(data,sz,copy,sz)==true; - } - - /** - * Test correctness of compressing and decompressing. - * Here we randomly assign a rate of exception (i.e. 1-alpha), - * and test different scale of normal/exception values. - */ - public void testAllDistribution() throws Exception { - initRandom(); - int sz = ForPostingsFormat.DEFAULT_BLOCK_SIZE; - int[] data = new int[sz]; - for (int i=0; i<=32; ++i) { // try to test every kinds of distribution - double alpha=gen.nextDouble(); // rate of normal value - for (int j=i; j<=32; ++j) { - createDistribution(data,sz,alpha,MASK[i],MASK[j]); - tryCompressAndDecompress(data, sz); - } - } - } - public void createDistribution(int[] data, int sz, double alpha, int masknorm, int maskexc) { - Integer[] buff= new Integer[sz]; - int i=0; - for (; i0 && (i)%16 == 0) - hex.append("\n"); - byte b=raw[i]; - hex.append(HEXES.charAt((b & 0xF0) >> 4)) - .append(HEXES.charAt((b & 0x0F))) - .append(" "); - } - return hex.toString(); - } - public static String getHex( int [] raw, int sz ) { - if ( raw == null ) { - return null; - } - final StringBuilder hex = new StringBuilder( 4 * raw.length ); - for ( int i=0; i0 && i%8 == 0) - hex.append("\n"); - hex.append(String.format(Locale.ENGLISH, "%08x ",raw[i])); - } - return hex.toString(); - } - static void eprintln(String format, Object... args) { - System.err.println(String.format(Locale.ENGLISH, format,args)); - } - static void println(String format, Object... args) { - System.out.println(String.format(Locale.ENGLISH, format,args)); - } - static void print(String format, Object... args) { - System.out.print(String.format(Locale.ENGLISH, format,args)); - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/pfor/PForFactory.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pfor/PForFactory.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/pfor/PForFactory.java (working copy) @@ -1,129 +0,0 @@ -package org.apache.lucene.codecs.pfor; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; - -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.codecs.sep.IntStreamFactory; -import org.apache.lucene.codecs.sep.IntIndexInput; -import org.apache.lucene.codecs.sep.IntIndexOutput; -import org.apache.lucene.codecs.intblock.FixedIntBlockIndexInput; -import org.apache.lucene.codecs.intblock.FixedIntBlockIndexOutput; - -/** - * Used to plug to PostingsReader/WriterBase. - * Encoder and decoder in lower layers are called by - * flushBlock() and readBlock() - */ - -public final class PForFactory extends IntStreamFactory { - - public PForFactory() { - } - - @Override - public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { - boolean success = false; - IndexOutput out = dir.createOutput(fileName, context); - try { - IntIndexOutput ret = new PForIndexOutput(out); - success = true; - return ret; - } finally { - if (!success) { - // For some cases (e.g. disk full), the IntIndexOutput may not be - // properly created. So we should close those opened files. - IOUtils.closeWhileHandlingException(out); - } - } - } - - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - return new PForIndexInput(dir.openInput(fileName, context)); - } - - /** - * Here we'll hold both input buffer and output buffer for - * encoder/decoder. - */ - private class PForIndexInput extends FixedIntBlockIndexInput { - - PForIndexInput(final IndexInput in) throws IOException { - super(in); - } - - class PForBlockReader implements FixedIntBlockIndexInput.BlockReader { - private final byte[] encoded; - private final int[] buffer; - private final IndexInput in; - private final IntBuffer encodedBuffer; - - PForBlockReader(final IndexInput in, final int[] buffer) { - // upperbound for encoded value should include(here header is not buffered): - // 1. blockSize of normal value (4x bytes); - // 2. blockSize of exception value (4x bytes); - this.encoded = new byte[PForPostingsFormat.DEFAULT_BLOCK_SIZE*8]; - this.in = in; - this.buffer = buffer; - this.encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); - } - - // TODO: implement public void skipBlock() {} ? - @Override - public void readBlock() throws IOException { - final int header = in.readInt(); - final int numBytes = PForUtil.getEncodedSize(header); - assert numBytes <= PForPostingsFormat.DEFAULT_BLOCK_SIZE*8; - in.readBytes(encoded,0,numBytes); - PForUtil.decompress(encodedBuffer,buffer,header); - } - } - - @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { - return new PForBlockReader(in,buffer); - } - } - - private class PForIndexOutput extends FixedIntBlockIndexOutput { - private final byte[] encoded; - private final IntBuffer encodedBuffer; - - PForIndexOutput(IndexOutput out) throws IOException { - super(out, PForPostingsFormat.DEFAULT_BLOCK_SIZE); - this.encoded = new byte[PForPostingsFormat.DEFAULT_BLOCK_SIZE*8]; - this.encodedBuffer=ByteBuffer.wrap(encoded).asIntBuffer(); - } - - @Override - protected void flushBlock() throws IOException { - final int header = PForUtil.compress(buffer,encodedBuffer); - final int numBytes = PForUtil.getEncodedSize(header); - // nocommit writeVInt instead? - out.writeInt(header); - out.writeBytes(encoded, numBytes); - } - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/pfor/ForFactory.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pfor/ForFactory.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/pfor/ForFactory.java (working copy) @@ -1,128 +0,0 @@ -package org.apache.lucene.codecs.pfor; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.IntBuffer; - -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.codecs.sep.IntStreamFactory; -import org.apache.lucene.codecs.sep.IntIndexInput; -import org.apache.lucene.codecs.sep.IntIndexOutput; -import org.apache.lucene.codecs.intblock.FixedIntBlockIndexInput; -import org.apache.lucene.codecs.intblock.FixedIntBlockIndexOutput; - -/** - * Used to plug to PostingsReader/WriterBase. - * Encoder and decoder in lower layers are called by - * flushBlock() and readBlock() - */ - -public final class ForFactory extends IntStreamFactory { - - public ForFactory() { - } - - @Override - public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { - boolean success = false; - IndexOutput out = dir.createOutput(fileName, context); - try { - IntIndexOutput ret = new ForIndexOutput(out); - success = true; - return ret; - } finally { - if (!success) { - // For some cases (e.g. disk full), the IntIndexOutput may not be - // properly created. So we should close those opened files. - IOUtils.closeWhileHandlingException(out); - } - } - } - - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - return new ForIndexInput(dir.openInput(fileName, context)); - } - - /** - * Here we'll hold both input buffer and output buffer for - * encoder/decoder. - */ - private class ForIndexInput extends FixedIntBlockIndexInput { - - ForIndexInput(final IndexInput in) throws IOException { - super(in); - } - - class ForBlockReader implements FixedIntBlockIndexInput.BlockReader { - private final byte[] encoded; - private final int[] buffer; - private final IndexInput in; - private final IntBuffer encodedBuffer; - - ForBlockReader(final IndexInput in, final int[] buffer) { - // upperbound for encoded value should include(here header is not buffered): - // blockSize of normal value when numFrameBits=32(4x bytes); - this.encoded = new byte[ForPostingsFormat.DEFAULT_BLOCK_SIZE*4]; - this.in = in; - this.buffer = buffer; - this.encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); - } - - // TODO: implement public void skipBlock() {} ? - @Override - public void readBlock() throws IOException { - final int header = in.readInt(); - final int numBytes = ForUtil.getEncodedSize(header); - assert numBytes <= ForPostingsFormat.DEFAULT_BLOCK_SIZE*4; - in.readBytes(encoded,0,numBytes); - ForUtil.decompress(encodedBuffer,buffer,header); - } - } - - @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { - return new ForBlockReader(in,buffer); - } - } - - private class ForIndexOutput extends FixedIntBlockIndexOutput { - private final byte[] encoded; - private final IntBuffer encodedBuffer; - - ForIndexOutput(IndexOutput out) throws IOException { - super(out,ForPostingsFormat.DEFAULT_BLOCK_SIZE); - this.encoded = new byte[ForPostingsFormat.DEFAULT_BLOCK_SIZE*4]; - this.encodedBuffer=ByteBuffer.wrap(encoded).asIntBuffer(); - } - - @Override - protected void flushBlock() throws IOException { - final int header = ForUtil.compress(buffer,encodedBuffer); - final int numBytes = ForUtil.getEncodedSize(header); - // nocommit writeVInt instead? - out.writeInt(header); - out.writeBytes(encoded, numBytes); - } - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/pfor/PForUtil.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pfor/PForUtil.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/pfor/PForUtil.java (working copy) @@ -1,343 +0,0 @@ -package org.apache.lucene.codecs.pfor; -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.nio.IntBuffer; -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * Encode all small values and exception pointers in normal area; - * Encode large values in exception area; - * Size per exception is variable, possibly: 1byte, 2bytes, or 4bytes - */ -public final class PForUtil extends ForUtil { - - protected static final int[] PER_EXCEPTION_SIZE = {1,2,4}; - - /** Compress given int[] into Integer buffer, with PFor format - * - * @param data uncompressed data - * @param intBuffer integer buffer to hold compressed data - * @return block header - */ - public static int compress(final int[] data, IntBuffer intBuffer) { - /** estimate minimum compress size to determine numFrameBits */ - int numBits=getNumBits(data); - if (numBits == 0) { - return compressDuplicateBlock(data,intBuffer); - } - - int size = data.length; - int[] excValues = new int[size]; - int excNum = 0, excLastPos = -1, excFirstPos = -1, excLastNonForcePos = -1; - - // num of exception until the last non-forced exception - int excNumBase = 0; - - // bytes per exception - int excBytes = 1; - - // bytes before exception area, e.g. header and normal area - int excByteOffset = 0; - - // the max value possible for current exception pointer, - // value of the first pointer is limited by header as 254 - // (first exception ranges from -1 ~ 254) - long maxChainFirst = 254; - long maxChain = maxChainFirst + 1; - - boolean conValue, conForce, conEnd; - int i=0; - - /** estimate exceptions */ - for (i=0; i= maxChain + excLastPos); // force exception - if (conValue || conForce) { - excValues[excNum++] = data[i]; - if (excLastPos == -1) { - maxChain = 1L<= maxChain + excLastPos); // force exception - conEnd = (excNum == excNumBase); // following forced ignored - if ((!conValue && !conForce) || conEnd) { - encodeNormalValue(intBuffer,i,data[i], numBits); - } else { - encodeNormalValue(intBuffer, excLastPos, i-excLastPos-1, numBits); - excNum++; - excLastPos = i; - } - } - } - - /** encode exception area */ - for (i=0; i> 8) & MASK[8]) + 1; - int excFirstPos = ((header >> 16) & MASK[8]) - 1; - int excBytes = PER_EXCEPTION_SIZE[(header >> 30) & MASK[2]]; - int numBits = ((header >> 24) & MASK[6]); - - decompressCore(intBuffer, data, numBits); - - patchException(intBuffer,data,excNum,excFirstPos,excBytes); - } - - /** - * Encode exception values into exception area. - * The width for each exception will be fixed as: - * 1, 2, or 4 byte(s). - */ - static void encodeExcValues(IntBuffer intBuffer, int[] values, int num, int perbytes, int byteOffset) { - if (num == 0) - return; - if (perbytes == 1) { - int curBytePos = byteOffset; - for (int i=0; i>> 8) & MASK[8]); - curPos = patch(data, curPos, (curInt >>> 16) & MASK[8]); - curPos = patch(data, curPos, (curInt >>> 24) & MASK[8]); - } - if (i>> j) & MASK[8]); - } - } - } else if (excBytes == 2) { // each exception consumes 2 bytes - for (i=0; i+1>> 16) & MASK[16]); - } - if (i pos; - return nextPos; - } - - /** - * Estimate best number of frame bits according to minimum compressed size. - * It will run 32 times. - */ - static int getNumBits(final int[] data) { - if (isAllEqual(data)) { - return 0; - } - int optBits=1; - int optSize=estimateCompressedSize(data,optBits); - for (int i=2; i<=32; ++i) { - int curSize=estimateCompressedSize(data,i); - if (curSize> 8) & MASK[8]) + 1; - } - public static int getFirstPos(int header) { - return ((header >> 16) & MASK[8]) - 1; - } - public static int getExcBytes(int header) { - return PER_EXCEPTION_SIZE[(header >> 30) & MASK[2]]; - } - public static int getNumBits(int header) { - return ((header >> 24) & MASK[6]); - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/pfor/ForUtil.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pfor/ForUtil.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/pfor/ForUtil.java (working copy) @@ -16,8 +16,19 @@ * limitations under the License. */ +import java.io.IOException; +import java.nio.LongBuffer; import java.nio.IntBuffer; +import java.nio.ByteBuffer; +import java.util.Arrays; +import org.apache.lucene.util.packed.PackedInts; +import org.apache.lucene.util.packed.PackedInts.Reader; +import org.apache.lucene.util.packed.PackedInts.Writer; +import org.apache.lucene.util.packed.PackedInts.Mutable; +import org.apache.lucene.util.packed.PackedInts.Encoder; +import org.apache.lucene.util.packed.PackedInts.Decoder; + /** * Encode all values in normal area with fixed bit width, * which is determined by the max value in this block. @@ -31,125 +42,58 @@ 0x01ffffff, 0x03ffffff, 0x07ffffff, 0x0fffffff, 0x1fffffff, 0x3fffffff, 0x7fffffff, 0xffffffff}; - /** Compress given int[] into Integer buffer, with For format - * - * @param data uncompressed data - * @param size num of ints to compress - * @param intBuffer integer buffer to hold compressed data - * @return encoded block byte size + /** Compress given int[] into output stream, with For format */ - public static int compress(final int[] data, IntBuffer intBuffer) { - int numBits=getNumBits(data); - if (numBits == 0) { - return compressDuplicateBlock(data,intBuffer); - } - - int size=data.length; - int encodedSize = (size*numBits+31)/32; + public static int compress(final LongBuffer data, LongBuffer packed) throws IOException { + int numBits=getNumBits(data.array()); - for (int i=0; i> 8) & MASK[6]); - decompressCore(intBuffer, data, numBits); - } - - /** - * IntBuffer will not be rewinded in this method, therefore - * caller should ensure that the position is set to the first - * encoded int before decoding. - */ - static void decompressCore(IntBuffer intBuffer, int[] data, int numBits) { - assert numBits<=32; - assert numBits>=0; - - // TODO: PackedIntsDecompress is hardewired to size==128 only - switch(numBits) { - case 0: PackedIntsDecompress.decode0(intBuffer, data); break; - case 1: PackedIntsDecompress.decode1(intBuffer, data); break; - case 2: PackedIntsDecompress.decode2(intBuffer, data); break; - case 3: PackedIntsDecompress.decode3(intBuffer, data); break; - case 4: PackedIntsDecompress.decode4(intBuffer, data); break; - case 5: PackedIntsDecompress.decode5(intBuffer, data); break; - case 6: PackedIntsDecompress.decode6(intBuffer, data); break; - case 7: PackedIntsDecompress.decode7(intBuffer, data); break; - case 8: PackedIntsDecompress.decode8(intBuffer, data); break; - case 9: PackedIntsDecompress.decode9(intBuffer, data); break; - case 10: PackedIntsDecompress.decode10(intBuffer, data); break; - case 11: PackedIntsDecompress.decode11(intBuffer, data); break; - case 12: PackedIntsDecompress.decode12(intBuffer, data); break; - case 13: PackedIntsDecompress.decode13(intBuffer, data); break; - case 14: PackedIntsDecompress.decode14(intBuffer, data); break; - case 15: PackedIntsDecompress.decode15(intBuffer, data); break; - case 16: PackedIntsDecompress.decode16(intBuffer, data); break; - case 17: PackedIntsDecompress.decode17(intBuffer, data); break; - case 18: PackedIntsDecompress.decode18(intBuffer, data); break; - case 19: PackedIntsDecompress.decode19(intBuffer, data); break; - case 20: PackedIntsDecompress.decode20(intBuffer, data); break; - case 21: PackedIntsDecompress.decode21(intBuffer, data); break; - case 22: PackedIntsDecompress.decode22(intBuffer, data); break; - case 23: PackedIntsDecompress.decode23(intBuffer, data); break; - case 24: PackedIntsDecompress.decode24(intBuffer, data); break; - case 25: PackedIntsDecompress.decode25(intBuffer, data); break; - case 26: PackedIntsDecompress.decode26(intBuffer, data); break; - case 27: PackedIntsDecompress.decode27(intBuffer, data); break; - case 28: PackedIntsDecompress.decode28(intBuffer, data); break; - case 29: PackedIntsDecompress.decode29(intBuffer, data); break; - case 30: PackedIntsDecompress.decode30(intBuffer, data); break; - case 31: PackedIntsDecompress.decode31(intBuffer, data); break; - case 32: PackedIntsDecompress.decode32(intBuffer, data); break; + if (numBits == 0) { + Arrays.fill(data.array(), (int)(packed.get(0)>>>32)); + return; } - } - static void encodeNormalValue(IntBuffer intBuffer, int pos, int value, int numBits) { - final int globalBitPos = numBits*pos; // position in bit stream - final int localBitPos = globalBitPos & 31; // position inside an int - int intPos = globalBitPos/32; // which integer to locate - setBufferIntBits(intBuffer, intPos, localBitPos, numBits, value); - if ((localBitPos + numBits) > 32) { // value does not fit in this int, fill tail - setBufferIntBits(intBuffer, intPos+1, 0, - (localBitPos+numBits-32), - (value >>> (32-localBitPos))); - } - } + PackedInts.Format format = PackedInts.fastestFormatAndBits(128, numBits, PackedInts.FASTEST).format; + PackedInts.Decoder decoder = PackedInts.getDecoder(format, PackedInts.VERSION_CURRENT, numBits); + int perIter = decoder.values(); + int iters = 128/perIter; + int nblocks = decoder.blocks()*iters; + assert 128 % perIter == 0; - static void setBufferIntBits(IntBuffer intBuffer, int intPos, int firstBitPos, int numBits, int value) { - assert (value & ~MASK[numBits]) == 0; - // safely discards those msb parts when firstBitPos+numBits>32 - intBuffer.put(intPos, - (intBuffer.get(intPos) & ~(MASK[numBits] << firstBitPos)) - | (value << firstBitPos)); + decoder.decode(packed, data, iters); } - /** - * Estimate best num of frame bits according to the largest value. - */ - static int getNumBits(final int[] data) { + static int getNumBits(final long[] data) { if (isAllEqual(data)) { return 0; } @@ -163,9 +107,9 @@ return optBits; } - protected static boolean isAllEqual(final int[] data) { + protected static boolean isAllEqual(final long[] data) { int len = data.length; - int v = data[0]; + long v = data[0]; for (int i=1; i 1; - this.maxBlockSize = maxBlockSize; - assert minBlockSize <= maxBlockSize; - } - - @Override - public String toString() { - return getName() + "(blocksize=" + DEFAULT_BLOCK_SIZE+ ")"; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - // TODO: implement a new PostingsWriterBase to improve skip-settings - PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new PForFactory()); - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, - postingsWriter, - minBlockSize, - maxBlockSize); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsWriter); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, - state.fieldInfos, - state.segmentInfo, - state.context, - new PForFactory(), - state.segmentSuffix); - - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - state.segmentSuffix, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsReader); - } - } - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/pfor/ForPostingsFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pfor/ForPostingsFormat.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/pfor/ForPostingsFormat.java (working copy) @@ -1,116 +0,0 @@ -package org.apache.lucene.codecs.pfor; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.codecs.BlockTreeTermsReader; -import org.apache.lucene.codecs.BlockTreeTermsWriter; -import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.PostingsReaderBase; -import org.apache.lucene.codecs.PostingsWriterBase; -import org.apache.lucene.codecs.TermsIndexReaderBase; -import org.apache.lucene.codecs.TermsIndexWriterBase; -import org.apache.lucene.codecs.sep.SepPostingsReader; -import org.apache.lucene.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** - * Pass ForFactory to a PostingsWriter/ReaderBase, and get - * customized postings format plugged. - */ -public final class ForPostingsFormat extends PostingsFormat { - private final int minBlockSize; - private final int maxBlockSize; - public final static int DEFAULT_BLOCK_SIZE = 128; - - public ForPostingsFormat() { - super("For"); - this.minBlockSize = BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE; - this.maxBlockSize = BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE; - } - - public ForPostingsFormat(int minBlockSize, int maxBlockSize) { - super("For"); - this.minBlockSize = minBlockSize; - assert minBlockSize > 1; - this.maxBlockSize = maxBlockSize; - assert minBlockSize <= maxBlockSize; - } - - @Override - public String toString() { - return getName() + "(blocksize=" + DEFAULT_BLOCK_SIZE + ")"; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - // TODO: implement a new PostingsWriterBase to improve skip-settings - PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new ForFactory()); - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, - postingsWriter, - minBlockSize, - maxBlockSize); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsWriter); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, - state.fieldInfos, - state.segmentInfo, - state.context, - new ForFactory(), - state.segmentSuffix); - - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - state.segmentSuffix, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(postingsReader); - } - } - } -} Index: lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java (revision 1367066) +++ lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.IntBuffer; +import java.nio.LongBuffer; import java.util.ArrayList; import java.util.List; @@ -78,14 +79,20 @@ private long posTermStartFP; private long payTermStartFP; - final int[] docDeltaBuffer; - final int[] freqBuffer; + final long[] docDeltaBuffer; + final long[] freqBuffer; + final LongBuffer docDeltaLBuffer; + final LongBuffer freqLBuffer; private int docBufferUpto; - final int[] posDeltaBuffer; - final int[] payloadLengthBuffer; - final int[] offsetStartDeltaBuffer; - final int[] offsetLengthBuffer; + final long[] posDeltaBuffer; + final long[] payloadLengthBuffer; + final long[] offsetStartDeltaBuffer; + final long[] offsetLengthBuffer; + final LongBuffer posDeltaLBuffer; + final LongBuffer payloadLengthLBuffer; + final LongBuffer offsetStartDeltaLBuffer; + final LongBuffer offsetLengthLBuffer; private int posBufferUpto; private byte[] payloadBytes; @@ -104,7 +111,7 @@ private int docCount; final byte[] encoded; - final IntBuffer encodedBuffer; + final LongBuffer encodedBuffer; private final BlockSkipWriter skipWriter; @@ -120,25 +127,32 @@ try { CodecUtil.writeHeader(docOut, DOC_CODEC, VERSION_CURRENT); if (state.fieldInfos.hasProx()) { - posDeltaBuffer = new int[blockSize]; + posDeltaBuffer = new long[blockSize]; + posDeltaLBuffer = LongBuffer.wrap(posDeltaBuffer); posOut = state.directory.createOutput(IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, BlockPostingsFormat.POS_EXTENSION), state.context); CodecUtil.writeHeader(posOut, POS_CODEC, VERSION_CURRENT); if (state.fieldInfos.hasPayloads()) { payloadBytes = new byte[128]; - payloadLengthBuffer = new int[blockSize]; + payloadLengthBuffer = new long[blockSize]; + payloadLengthLBuffer = LongBuffer.wrap(payloadLengthBuffer); } else { payloadBytes = null; payloadLengthBuffer = null; + payloadLengthLBuffer = null; } if (state.fieldInfos.hasOffsets()) { - offsetStartDeltaBuffer = new int[blockSize]; - offsetLengthBuffer = new int[blockSize]; + offsetStartDeltaBuffer = new long[blockSize]; + offsetLengthBuffer = new long[blockSize]; + offsetStartDeltaLBuffer = LongBuffer.wrap(offsetStartDeltaBuffer); + offsetLengthLBuffer = LongBuffer.wrap(offsetLengthBuffer); } else { offsetStartDeltaBuffer = null; offsetLengthBuffer = null; + offsetStartDeltaLBuffer = null; + offsetLengthLBuffer = null; } if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) { @@ -152,6 +166,10 @@ offsetStartDeltaBuffer = null; offsetLengthBuffer = null; payloadBytes = null; + posDeltaLBuffer = null; + payloadLengthLBuffer = null; + offsetStartDeltaLBuffer = null; + offsetLengthLBuffer = null; } this.payOut = payOut; this.posOut = posOut; @@ -162,8 +180,10 @@ } } - docDeltaBuffer = new int[blockSize]; - freqBuffer = new int[blockSize]; + docDeltaBuffer = new long[blockSize]; + freqBuffer = new long[blockSize]; + docDeltaLBuffer = LongBuffer.wrap(docDeltaBuffer); + freqLBuffer = LongBuffer.wrap(freqBuffer); skipWriter = new BlockSkipWriter(blockSize, maxSkipLevels, @@ -172,8 +192,8 @@ posOut, payOut); - encoded = new byte[blockSize*4 + 4]; - encodedBuffer = ByteBuffer.wrap(encoded).asIntBuffer(); + encoded = new byte[blockSize*4]; + encodedBuffer = ByteBuffer.wrap(encoded).asLongBuffer(); } @Override @@ -210,9 +230,8 @@ skipWriter.resetSkip(); } - private void writeBlock(int[] buffer, IndexOutput out) throws IOException { + private void writeBlock(LongBuffer buffer, IndexOutput out) throws IOException { final int header = ForUtil.compress(buffer, encodedBuffer); - //System.out.println(" block has " + numBytes + " bytes"); out.writeVInt(header); out.writeBytes(encoded, ForUtil.getEncodedSize(header)); } @@ -278,12 +297,12 @@ if (DEBUG) { System.out.println(" write docDelta block @ fp=" + docOut.getFilePointer()); } - writeBlock(docDeltaBuffer, docOut); + writeBlock(docDeltaLBuffer, docOut); if (fieldHasFreqs) { if (DEBUG) { System.out.println(" write freq block @ fp=" + docOut.getFilePointer()); } - writeBlock(freqBuffer, docOut); + writeBlock(freqLBuffer, docOut); } docBufferUpto = 0; } @@ -327,17 +346,17 @@ if (DEBUG) { System.out.println(" write pos bulk block @ fp=" + posOut.getFilePointer()); } - writeBlock(posDeltaBuffer, posOut); + writeBlock(posDeltaLBuffer, posOut); if (fieldHasPayloads) { - writeBlock(payloadLengthBuffer, payOut); + writeBlock(payloadLengthLBuffer, payOut); payOut.writeVInt(payloadByteUpto); payOut.writeBytes(payloadBytes, 0, payloadByteUpto); payloadByteUpto = 0; } if (fieldHasOffsets) { - writeBlock(offsetStartDeltaBuffer, payOut); - writeBlock(offsetLengthBuffer, payOut); + writeBlock(offsetStartDeltaLBuffer, payOut); + writeBlock(offsetLengthLBuffer, payOut); } posBufferUpto = 0; } @@ -400,8 +419,8 @@ // vInt encode the remaining doc deltas and freqs: for(int i=0;i