Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) @@ -0,0 +1,428 @@ +package org.apache.lucene.util.packed; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.packed.PackedInts.ReaderIterator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This class is similar to {@link Packed64} except that it trades space for + * speed by ensuring that a single block needs to be read/written in order to + * read/write a value. + */ +abstract class Packed64SingleBlock extends PackedInts.MutableImpl { + + private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {1, 2, 3, 4, + 5, 6, 7, 9, 10, 12, 21}; + private static final long[][] WRITE_MASKS = new long[22][]; + private static final int[][] SHIFTS = new int[22][]; + static { + for (int bpv : SUPPORTED_BITS_PER_VALUE) { + initMasks(bpv); + } + } + + protected static void initMasks(int bpv) { + int valuesPerBlock = Long.SIZE / bpv; + long[] writeMasks = new long[valuesPerBlock]; + int[] shifts = new int[valuesPerBlock]; + long bits = (1L << bpv) - 1; + for (int i = 0; i < valuesPerBlock; ++i) { + shifts[i] = bpv * i; + writeMasks[i] = ~(bits << shifts[i]); + } + WRITE_MASKS[bpv] = writeMasks; + SHIFTS[bpv] = shifts; + } + + public static Packed64SingleBlock create(int valueCount, int bitsPerValue) { + switch (bitsPerValue) { + case 1: + return new Packed64SingleBlock1(valueCount); + case 2: + return new Packed64SingleBlock2(valueCount); + case 3: + return new Packed64SingleBlock3(valueCount); + case 4: + return new Packed64SingleBlock4(valueCount); + case 5: + return new Packed64SingleBlock5(valueCount); + case 6: + return new Packed64SingleBlock6(valueCount); + case 7: + return new Packed64SingleBlock7(valueCount); + case 9: + return new Packed64SingleBlock9(valueCount); + case 10: + return new Packed64SingleBlock10(valueCount); + case 12: + return new Packed64SingleBlock12(valueCount); + case 21: + return new Packed64SingleBlock21(valueCount); + default: + throw new IllegalArgumentException("Unsupported bitsPerValue: " + + bitsPerValue); + } + } + + public static boolean isSupported(int bitsPerValue) { + return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + public static float overheadPerValue(int bitsPerValue) { + int valuesPerBlock = 64 / bitsPerValue; + int overhead = 64 % bitsPerValue; + return (float) overhead / valuesPerBlock; + } + + protected final long[] blocks; + protected final int valuesPerBlock; + protected final int[] shifts; + protected final long[] writeMasks; + protected final long readMask; + + Packed64SingleBlock(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + valuesPerBlock = Long.SIZE / bitsPerValue; + blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; + shifts = SHIFTS[bitsPerValue]; + writeMasks = WRITE_MASKS[bitsPerValue]; + readMask = ~writeMasks[0]; + } + + private static int requiredCapacity(int valueCount, int valuesPerBlock) { + return valueCount / valuesPerBlock + + (valueCount % valuesPerBlock == 0 ? 0 : 1); + } + + protected int blockOffset(int offset) { + return offset / valuesPerBlock; + } + + protected int offsetInBlock(int offset) { + return offset % valuesPerBlock; + } + + @Override + public long get(int index) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + return (blocks[o] >> shifts[b]) & readMask; + } + + @Override + public void set(int index, long value) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + blocks[o] = (blocks[o] & writeMasks[b]) | (value << shifts[b]); + } + + protected long nextBlock(ReaderIterator iterator) throws IOException { + long l = 0L; + for (int i = 0; i < valuesPerBlock; ++i) { + l |= iterator.next() << shifts[i]; + } + return l; + } + + @Override + public int set(int fromIndex, int length, ReaderIterator iterator) + throws IOException { + final int totalCount = Math.min(length, iterator.size() - iterator.ord() - 1); + int count = totalCount; + + while (offsetInBlock(fromIndex) != 0 && count > 0) { + set(fromIndex++, iterator.next()); + --count; + } + + int nBlocks = blockOffset(count); + for (int block = blockOffset(fromIndex), toBlock = block + nBlocks; block < toBlock; ++block) { + blocks[block] = nextBlock(iterator); + } + final int done = valuesPerBlock * nBlocks; + + int sets = super.set(fromIndex + done, count - done, iterator); + assert sets < valuesPerBlock; + + return totalCount; + } + + @Override + public void clear() { + Arrays.fill(blocks, 0L); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } + + // Specialisations that allow the JVM to optimize computation of the block + // offset as well as the offset in block + + static final class Packed64SingleBlock21 extends Packed64SingleBlock { + + Packed64SingleBlock21(int valueCount) { + super(valueCount, 21); + assert valuesPerBlock == 3; + } + + @Override + protected int blockOffset(int offset) { + return offset / 3; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 3; + } + + @Override + protected long nextBlock(ReaderIterator it) throws IOException { + return it.next() | (it.next() << shifts[1]) | (it.next() << shifts[2]); + } + + } + + static final class Packed64SingleBlock12 extends Packed64SingleBlock { + + Packed64SingleBlock12(int valueCount) { + super(valueCount, 12); + assert valuesPerBlock == 5; + } + + @Override + protected int blockOffset(int offset) { + return offset / 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 5; + } + + @Override + protected long nextBlock(ReaderIterator it) throws IOException { + return it.next() | (it.next() << shifts[1]) | (it.next() << shifts[2]) + | (it.next() << shifts[3]) | (it.next() << shifts[4]); + } + } + + static final class Packed64SingleBlock10 extends Packed64SingleBlock { + + Packed64SingleBlock10(int valueCount) { + super(valueCount, 10); + assert valuesPerBlock == 6; + } + + @Override + protected int blockOffset(int offset) { + return offset / 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 6; + } + + @Override + protected long nextBlock(ReaderIterator it) throws IOException { + return it.next() | (it.next() << shifts[1]) | (it.next() << shifts[2]) + | (it.next() << shifts[3]) | (it.next() << shifts[4]) + | (it.next() << shifts[5]); + } + } + + static final class Packed64SingleBlock9 extends Packed64SingleBlock { + + Packed64SingleBlock9(int valueCount) { + super(valueCount, 9); + assert valuesPerBlock == 7; + } + + @Override + protected int blockOffset(int offset) { + return offset / 7; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 7; + } + + @Override + protected long nextBlock(ReaderIterator it) throws IOException { + return it.next() | (it.next() << shifts[1]) | (it.next() << shifts[2]) + | (it.next() << shifts[3]) | (it.next() << shifts[4]) + | (it.next() << shifts[5]) | (it.next() << shifts[6]); + } + + } + + static final class Packed64SingleBlock7 extends Packed64SingleBlock { + + Packed64SingleBlock7(int valueCount) { + super(valueCount, 7); + assert valuesPerBlock == 9; + } + + @Override + protected int blockOffset(int offset) { + return offset / 9; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 9; + } + + @Override + protected long nextBlock(ReaderIterator it) throws IOException { + return it.next() | (it.next() << shifts[1]) | (it.next() << shifts[2]) + | (it.next() << shifts[3]) | (it.next() << shifts[4]) + | (it.next() << shifts[5]) | (it.next() << shifts[6]) + | (it.next() << shifts[7]) | (it.next() << shifts[8]); + } + + } + + static final class Packed64SingleBlock6 extends Packed64SingleBlock { + + Packed64SingleBlock6(int valueCount) { + super(valueCount, 6); + assert valuesPerBlock == 10; + } + + @Override + protected int blockOffset(int offset) { + return offset / 10; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 10; + } + + } + + static final class Packed64SingleBlock5 extends Packed64SingleBlock { + + Packed64SingleBlock5(int valueCount) { + super(valueCount, 5); + assert valuesPerBlock == 12; + } + + @Override + protected int blockOffset(int offset) { + return offset / 12; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 12; + } + + } + + static final class Packed64SingleBlock4 extends Packed64SingleBlock { + + Packed64SingleBlock4(int valueCount) { + super(valueCount, 4); + assert valuesPerBlock == 16; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 4; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 15; + } + + } + + static final class Packed64SingleBlock3 extends Packed64SingleBlock { + + Packed64SingleBlock3(int valueCount) { + super(valueCount, 3); + assert valuesPerBlock == 21; + } + + @Override + protected int blockOffset(int offset) { + return offset / 21; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 21; + } + + } + + static final class Packed64SingleBlock2 extends Packed64SingleBlock { + + Packed64SingleBlock2(int valueCount) { + super(valueCount, 2); + assert valuesPerBlock == 32; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 31; + } + + } + + static final class Packed64SingleBlock1 extends Packed64SingleBlock { + + Packed64SingleBlock1(int valueCount) { + super(valueCount, 1); + assert valuesPerBlock == 64; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 63; + } + + } +} \ No newline at end of file Index: lucene/core/src/java/org/apache/lucene/util/packed/SequentialPackedReaderIterator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/SequentialPackedReaderIterator.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/SequentialPackedReaderIterator.java (révision 0) @@ -0,0 +1,49 @@ +package org.apache.lucene.util.packed; + +import java.io.IOException; + +import org.apache.lucene.store.DataInput; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +final class SequentialPackedReaderIterator extends AbstractPackedReaderIterator { + + private final DataInput in; + + public SequentialPackedReaderIterator(int bitsPerValue, int valueCount, + DataInput in) throws IOException { + super(bitsPerValue, valueCount); + this.in = in; + } + + @Override + public long advance(int ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void close() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + protected DataInput getDataInput() { + return in; + } + +} Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java (copie de travail) @@ -28,8 +28,7 @@ * @lucene.internal */ -class Direct32 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { +class Direct32 extends PackedInts.MutableImpl { private int[] values; private static final int BITS_PER_VALUE = 32; Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java (copie de travail) @@ -28,8 +28,7 @@ * @lucene.internal */ -class Direct16 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { +class Direct16 extends PackedInts.MutableImpl { private short[] values; private static final int BITS_PER_VALUE = 16; Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed16ThreeBlocks.java (révision 0) @@ -0,0 +1,67 @@ +package org.apache.lucene.util.packed; + +import java.util.Arrays; + +import org.apache.lucene.util.RamUsageEstimator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +final class Packed16ThreeBlocks extends PackedInts.MutableImpl { + + public static final int MAX_SIZE = Integer.MAX_VALUE / 3; + + private final short[] blocks; + + Packed16ThreeBlocks(int valueCount) { + super(valueCount, 48); + if (valueCount > MAX_SIZE) { + throw new ArrayIndexOutOfBoundsException("MAX_SIZE exceeded"); + } + this.blocks = new short[3 * valueCount]; + } + + @Override + public long get(int index) { + final int o = index * 3; + return (blocks[o] & 0xffffL) << 32 | (blocks[o+1] & 0xffffL) << 16 | (blocks[o+2] & 0xffffL); + } + + @Override + public void set(int index, long value) { + final int o = index * 3; + blocks[o] = (short) (value >> 32); + blocks[o+1] = (short) (value >> 16); + blocks[o+2] = (short) value; + } + + @Override + public void clear() { + Arrays.fill(blocks, (short) 0); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } + +} Index: lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (copie de travail) @@ -17,76 +17,30 @@ * limitations under the License. */ +import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; import java.io.IOException; -final class PackedReaderIterator implements PackedInts.ReaderIterator { - private long pending; - private int pendingBitsLeft; +final class PackedReaderIterator extends AbstractPackedReaderIterator { + private final IndexInput in; - private final int bitsPerValue; - private final int valueCount; - private int position = -1; - // masks[n-1] masks for bottom n bits - private final long[] masks; - public PackedReaderIterator(int bitsPerValue, int valueCount, IndexInput in) throws IOException { - - this.valueCount = valueCount; - this.bitsPerValue = bitsPerValue; - + super(bitsPerValue, valueCount); this.in = in; - masks = new long[bitsPerValue]; - - long v = 1; - for (int i = 0; i < bitsPerValue; i++) { - v *= 2; - masks[i] = v - 1; - } } - public int getBitsPerValue() { - return bitsPerValue; + @Override + protected DataInput getDataInput() { + return in; } - public int size() { - return valueCount; - } - - public long next() throws IOException { - if (pendingBitsLeft == 0) { - pending = in.readLong(); - pendingBitsLeft = 64; - } - - final long result; - if (pendingBitsLeft >= bitsPerValue) { // not split - result = (pending >> (pendingBitsLeft - bitsPerValue)) & masks[bitsPerValue-1]; - pendingBitsLeft -= bitsPerValue; - } else { // split - final int bits1 = bitsPerValue - pendingBitsLeft; - final long result1 = (pending & masks[pendingBitsLeft-1]) << bits1; - pending = in.readLong(); - final long result2 = (pending >> (64 - bits1)) & masks[bits1-1]; - pendingBitsLeft = 64 + pendingBitsLeft - bitsPerValue; - result = result1 | result2; - } - - ++position; - return result; - } - public void close() throws IOException { in.close(); } - public int ord() { - return position; - } - public long advance(final int ord) throws IOException{ assert ord < valueCount : "ord must be less than valueCount"; assert ord > position : "ord must be greater than the current position"; Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java (copie de travail) @@ -28,8 +28,7 @@ * @lucene.internal */ -class Direct64 extends PackedInts.ReaderImpl - implements PackedInts.Mutable { +class Direct64 extends PackedInts.MutableImpl { private long[] values; private static final int BITS_PER_VALUE = 64; Index: lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (copie de travail) @@ -38,6 +38,10 @@ public class PackedInts { + public static final float FAST = 1f; + public static final float COMPACT = 0f; + public static final float DEFAULT = 0.2f; + private final static String CODEC_NAME = "PackedInts"; private final static int VERSION_START = 0; private final static int VERSION_CURRENT = VERSION_START; @@ -117,9 +121,20 @@ void set(int index, long value); /** + * Set a range of values based on a {@link ReaderIterator} and return the + * number of values that have been set. + * + * @param fromIndex the start index (inclusive) + * @param length the maximum number of integers to set + * @param iterator the integers to set + * @return the number of ints which have been set + * @throws IOException + */ + int set(int fromIndex, int length, ReaderIterator iterator) throws IOException; + + /** * Sets all values to 0. - */ - + */ void clear(); } @@ -145,10 +160,6 @@ return valueCount; } - public long getMaxValue() { // Convenience method - return maxValue(bitsPerValue); - } - public Object getArray() { return null; } @@ -158,6 +169,24 @@ } } + public static abstract class MutableImpl extends ReaderImpl implements Mutable { + + protected MutableImpl(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + } + + @Override + public int set(int fromIndex, int length, ReaderIterator iterator) throws IOException { + final int count = Math.min(length, iterator.size() - iterator.ord() - 1); + final int maxIndex = fromIndex + count; + for (int i = fromIndex; i < maxIndex; ++i) { + set(i, iterator.next()); + } + return count; + } + + } + /** A write-once Writer. * @lucene.internal */ @@ -182,32 +211,76 @@ public abstract void finish() throws IOException; } + private static Reader copyData(int bitsPerValue, int valueCount, + DataInput in, Mutable mutable) throws IOException { + ReaderIterator it = new SequentialPackedReaderIterator(bitsPerValue, valueCount, in); + mutable.set(0, valueCount, it); + return mutable; + } + /** * Retrieve PackedInt data from the DataInput and return a packed int * structure based on it. + * + * Positive values of acceptableOverheadRatio will trade space + * for speed by selecting a faster but potentially less memory-efficient + * implementation. An acceptableOverheadRatio of + * {@link PackedInts#COMPACT} will make sure that the most memory-efficient + * implementation is selected whereas {@link PackedInts#FAST} will make sure + * that the fastest implementation is selected. + * * @param in positioned at the beginning of a stored packed int structure. + * @param acceptableOverheadRatio an acceptable overhead ratio per value * @return a read only random access capable array of positive integers. * @throws IOException if the structure could not be retrieved. * @lucene.internal */ - public static Reader getReader(DataInput in) throws IOException { + public static Reader getReader(DataInput in, float acceptableOverheadRatio) throws IOException { CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START); final int bitsPerValue = in.readVInt(); assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue; final int valueCount = in.readVInt(); - switch (bitsPerValue) { - case 8: + acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); + acceptableOverheadRatio = Math.min(FAST, acceptableOverheadRatio); + float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits + + int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + + if (bitsPerValue == Byte.SIZE) { return new Direct8(in, valueCount); - case 16: + } else if (bitsPerValue < Byte.SIZE && maxBitsPerValue >= Byte.SIZE) { + return copyData(bitsPerValue, valueCount, in, new Direct8(valueCount)); + } else if (bitsPerValue == Short.SIZE) { return new Direct16(in, valueCount); - case 32: + } else if (bitsPerValue < Short.SIZE && maxBitsPerValue >= Short.SIZE) { + return copyData(bitsPerValue, valueCount, in, new Direct16(valueCount)); + } else if (bitsPerValue == Integer.SIZE) { return new Direct32(in, valueCount); - case 64: + } else if (bitsPerValue < Integer.SIZE && maxBitsPerValue >= Integer.SIZE) { + return copyData(bitsPerValue, valueCount, in, new Direct32(valueCount)); + } else if (bitsPerValue == Long.SIZE) { return new Direct64(in, valueCount); - default: - if (Constants.JRE_IS_64BIT || bitsPerValue >= 32) { + } else if (bitsPerValue < Long.SIZE && maxBitsPerValue >= Long.SIZE) { + return copyData(bitsPerValue, valueCount, in, new Direct64(valueCount)); + } else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) { + return copyData(bitsPerValue, valueCount, in, new Packed8ThreeBlocks(valueCount)); + } else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) { + return copyData(bitsPerValue, valueCount, in, new Packed16ThreeBlocks(valueCount)); + } else { + if (bitsPerValue >= 32) { return new Packed64(in, valueCount, bitsPerValue); + } else if (Constants.JRE_IS_64BIT) { + for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { + if (Packed64SingleBlock.isSupported(bpv)) { + float overhead = Packed64SingleBlock.overheadPerValue(bpv); + float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; + if (overhead <= acceptableOverhead) { + return copyData(bitsPerValue, valueCount, in, Packed64SingleBlock.create(valueCount, bpv)); + } + } + } + return new Packed64(in, valueCount, bitsPerValue); } else { return new Packed32(in, valueCount, bitsPerValue); } @@ -250,28 +323,57 @@ * Create a packed integer array with the given amount of values initialized * to 0. the valueCount and the bitsPerValue cannot be changed after creation. * All Mutables known by this factory are kept fully in RAM. - * @param valueCount the number of elements. - * @param bitsPerValue the number of bits available for any given value. - * @return a mutable packed integer array. + * + * Positive values of acceptableOverheadRatio will trade space + * for speed by selecting a faster but potentially less memory-efficient + * implementation. An acceptableOverheadRatio of + * {@link PackedInts#COMPACT} will make sure that the most memory-efficient + * implementation is selected whereas {@link PackedInts#FAST} will make sure + * that the fastest implementation is selected. + * + * @param valueCount the number of elements + * @param bitsPerValue the number of bits available for any given value + * @param acceptableOverheadRatio an acceptable overhead ratio per value + * @return a mutable packed integer array * @throws java.io.IOException if the Mutable could not be created. With the * current implementations, this never happens, but the method * signature allows for future persistence-backed Mutables. * @lucene.internal */ - public static Mutable getMutable( - int valueCount, int bitsPerValue) { - switch (bitsPerValue) { - case 8: + public static Mutable getMutable(int valueCount, + int bitsPerValue, float acceptableOverheadRatio) { + acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); + acceptableOverheadRatio = Math.min(FAST, acceptableOverheadRatio); + float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits + + int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + + if (bitsPerValue <= Byte.SIZE && maxBitsPerValue >= Byte.SIZE) { return new Direct8(valueCount); - case 16: + } else if (bitsPerValue <= Short.SIZE && maxBitsPerValue >= Short.SIZE) { return new Direct16(valueCount); - case 32: + } else if (bitsPerValue <= Integer.SIZE && maxBitsPerValue >= Integer.SIZE) { return new Direct32(valueCount); - case 64: + } else if (bitsPerValue <= Long.SIZE && maxBitsPerValue >= Long.SIZE) { return new Direct64(valueCount); - default: - if (Constants.JRE_IS_64BIT || bitsPerValue >= 32) { + } else if (valueCount <= Packed8ThreeBlocks.MAX_SIZE && bitsPerValue <= 24 && maxBitsPerValue >= 24) { + return new Packed8ThreeBlocks(valueCount); + } else if (valueCount <= Packed16ThreeBlocks.MAX_SIZE && bitsPerValue <= 48 && maxBitsPerValue >= 48) { + return new Packed16ThreeBlocks(valueCount); + } else { + if (bitsPerValue >= 32) { return new Packed64(valueCount, bitsPerValue); + } else if (Constants.JRE_IS_64BIT) { + for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { + if (Packed64SingleBlock.isSupported(bpv)) { + float overhead = Packed64SingleBlock.overheadPerValue(bpv); + float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; + if (overhead <= acceptableOverhead) { + return Packed64SingleBlock.create(valueCount, bpv); + } + } + } + return new Packed64(valueCount, bitsPerValue); } else { return new Packed32(valueCount, bitsPerValue); } @@ -321,26 +423,4 @@ public static long maxValue(int bitsPerValue) { return bitsPerValue == 64 ? Long.MAX_VALUE : ~(~0L << bitsPerValue); } - - /** Rounds bitsPerValue up to 8, 16, 32 or 64. */ - public static int getNextFixedSize(int bitsPerValue) { - if (bitsPerValue <= 8) { - return 8; - } else if (bitsPerValue <= 16) { - return 16; - } else if (bitsPerValue <= 32) { - return 32; - } else { - return 64; - } - } - - /** Possibly wastes some storage in exchange for faster lookups */ - public static int getRoundedFixedSize(int bitsPerValue) { - if (bitsPerValue > 58 || (bitsPerValue < 32 && bitsPerValue > 29)) { // 10% space-waste is ok - return getNextFixedSize(bitsPerValue); - } else { - return bitsPerValue; - } - } } Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java (copie de travail) @@ -32,7 +32,7 @@ * sacrificing code clarity to achieve better performance. */ -class Packed32 extends PackedInts.ReaderImpl implements PackedInts.Mutable { +class Packed32 extends PackedInts.MutableImpl { static final int BLOCK_SIZE = 32; // 32 = int, 64 = long static final int BLOCK_BITS = 5; // The #bits representing BLOCK_SIZE static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE Index: lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (copie de travail) @@ -1,5 +1,9 @@ package org.apache.lucene.util.packed; +import java.io.IOException; + +import org.apache.lucene.util.packed.PackedInts.ReaderIterator; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -28,22 +32,14 @@ private long currentMaxValue; private PackedInts.Mutable current; - private final boolean roundFixedSize; + private final float acceptableOverheadRatio; - public GrowableWriter(int startBitsPerValue, int valueCount, boolean roundFixedSize) { - this.roundFixedSize = roundFixedSize; - current = PackedInts.getMutable(valueCount, getSize(startBitsPerValue)); + public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) { + this.acceptableOverheadRatio = acceptableOverheadRatio; + current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio); currentMaxValue = PackedInts.maxValue(current.getBitsPerValue()); } - private final int getSize(int bpv) { - if (roundFixedSize) { - return PackedInts.getNextFixedSize(bpv); - } else { - return bpv; - } - } - public long get(int index) { return current.get(index); } @@ -78,7 +74,7 @@ currentMaxValue *= 2; } final int valueCount = size(); - PackedInts.Mutable next = PackedInts.getMutable(valueCount, getSize(bpv)); + PackedInts.Mutable next = PackedInts.getMutable(valueCount, bpv, acceptableOverheadRatio); for(int i=0;i MAX_SIZE) { + throw new ArrayIndexOutOfBoundsException("MAX_SIZE exceeded"); + } + this.blocks = new byte[3 * valueCount]; + } + + @Override + public long get(int index) { + final int o = index * 3; + return (blocks[o] & 0xffL) << 16 | (blocks[o+1] & 0xffL) << 8 | (blocks[o+2] & 0xffL); + } + + @Override + public void set(int index, long value) { + final int o = index * 3; + blocks[o+2] = (byte) value; + blocks[o+1] = (byte) (value >> 8); + blocks[o] = (byte) (value >> 16); + } + + @Override + public void clear() { + Arrays.fill(blocks, (byte) 0); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(bitsPerValue=" + bitsPerValue + + ", size=" + size() + ", elements.length=" + blocks.length + ")"; + } + +} Index: lucene/core/src/java/org/apache/lucene/util/packed/AbstractPackedReaderIterator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/AbstractPackedReaderIterator.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/AbstractPackedReaderIterator.java (révision 0) @@ -0,0 +1,88 @@ +package org.apache.lucene.util.packed; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.DataInput; + +abstract class AbstractPackedReaderIterator implements PackedInts.ReaderIterator { + protected long pending; + protected int pendingBitsLeft; + protected final int bitsPerValue; + protected final int valueCount; + protected int position = -1; + + // masks[n-1] masks for bottom n bits + private final long[] masks; + + public AbstractPackedReaderIterator(int bitsPerValue, int valueCount) + throws IOException { + + this.valueCount = valueCount; + this.bitsPerValue = bitsPerValue; + + masks = new long[bitsPerValue]; + + long v = 1; + for (int i = 0; i < bitsPerValue; i++) { + v *= 2; + masks[i] = v - 1; + } + } + + protected abstract DataInput getDataInput(); + + public final int getBitsPerValue() { + return bitsPerValue; + } + + public final int size() { + return valueCount; + } + + public final long next() throws IOException { + final DataInput in = getDataInput(); + + if (pendingBitsLeft == 0) { + pending = in.readLong(); + pendingBitsLeft = 64; + } + + final long result; + if (pendingBitsLeft >= bitsPerValue) { // not split + result = (pending >> (pendingBitsLeft - bitsPerValue)) & masks[bitsPerValue-1]; + pendingBitsLeft -= bitsPerValue; + } else { // split + final int bits1 = bitsPerValue - pendingBitsLeft; + final long result1 = (pending & masks[pendingBitsLeft-1]) << bits1; + pending = in.readLong(); + final long result2 = (pending >> (64 - bits1)) & masks[bits1-1]; + pendingBitsLeft = 64 + pendingBitsLeft - bitsPerValue; + result = result1 | result2; + } + + ++position; + return result; + } + + public final int ord() { + return position; + } + +} Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (copie de travail) @@ -115,17 +115,19 @@ * {@link Writer}. A call to {@link Writer#finish(int)} will release * all internally used resources and frees the memory tracking * reference. - * @param fasterButMoreRam whether packed ints for docvalues should be optimized for speed by rounding up the bytes - * used for a value to either 8, 16, 32 or 64 bytes. This option is only applicable for - * docvalues of type {@link Type#BYTES_FIXED_SORTED} and {@link Type#BYTES_VAR_SORTED}. + * @param acceptableOverheadRatio + * how to trade space for speed. This option is only applicable for + * docvalues of type {@link Type#BYTES_FIXED_SORTED} and + * {@link Type#BYTES_VAR_SORTED}. * @param context I/O Context * @return a new {@link Writer} instance * @throws IOException * if the files for the writer can not be created. + * @see PackedInts#getReader(org.apache.lucene.store.DataInput, float) */ public static DocValuesConsumer getWriter(Directory dir, String id, Mode mode, boolean fixedSize, Comparator sortComparator, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? @@ -139,7 +141,7 @@ } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); + return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, acceptableOverheadRatio); } } else { if (mode == Mode.STRAIGHT) { @@ -147,7 +149,7 @@ } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); + return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, acceptableOverheadRatio); } } @@ -374,32 +376,32 @@ protected int lastDocId = -1; protected int[] docToEntry; protected final BytesRefHash hash; - protected final boolean fasterButMoreRam; + protected final float acceptableOverheadRatio; protected long maxBytes = 0; protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Counter bytesUsed, IOContext context, Type type) throws IOException { this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( - ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, false, type); + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, PackedInts.DEFAULT, type); } protected DerefBytesWriterBase(Directory dir, String id, String codecName, - int codecVersion, Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type) + int codecVersion, Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) throws IOException { this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( - ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, fasterButMoreRam,type); + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, acceptableOverheadRatio,type); } protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Allocator allocator, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type) throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) throws IOException { super(dir, id, codecName, codecVersion, bytesUsed, context, type); hash = new BytesRefHash(new ByteBlockPool(allocator), BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray( BytesRefHash.DEFAULT_CAPACITY, bytesUsed)); docToEntry = new int[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); - this.fasterButMoreRam = fasterButMoreRam; + this.acceptableOverheadRatio = acceptableOverheadRatio; } protected static int writePrefixLength(DataOutput datOut, BytesRef bytes) @@ -498,7 +500,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, int[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - bitsRequired(maxValue)); + PackedInts.bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -522,7 +524,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, long[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - bitsRequired(maxValue)); + PackedInts.bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -542,11 +544,6 @@ } w.finish(); } - - protected int bitsRequired(long maxValue){ - return fasterButMoreRam ? - PackedInts.getNextFixedSize(PackedInts.bitsRequired(maxValue)) : PackedInts.bitsRequired(maxValue); - } } @@ -578,8 +575,8 @@ this.pagedBytes.copy(datIn, bytesToRead); data = pagedBytes.freeze(true); this.idxIn = idxIn; - ordToOffsetIndex = hasOffsets ? PackedInts.getReader(idxIn) : null; - docToOrdIndex = PackedInts.getReader(idxIn); + ordToOffsetIndex = hasOffsets ? PackedInts.getReader(idxIn, PackedInts.DEFAULT) : null; + docToOrdIndex = PackedInts.getReader(idxIn, PackedInts.DEFAULT); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarDerefBytesImpl.java (copie de travail) @@ -116,7 +116,7 @@ throws IOException { super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), totalBytes, Type.BYTES_VAR_DEREF); - addresses = PackedInts.getReader(idxIn); + addresses = PackedInts.getReader(idxIn, PackedInts.DEFAULT); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarStraightBytesImpl.java (copie de travail) @@ -258,7 +258,7 @@ public VarStraightSource(IndexInput datIn, IndexInput idxIn) throws IOException { super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), idxIn.readVLong(), Type.BYTES_VAR_STRAIGHT); - addresses = PackedInts.getReader(idxIn); + addresses = PackedInts.getReader(idxIn, PackedInts.DEFAULT); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (copie de travail) @@ -57,8 +57,8 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_FIXED_SORTED); + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_FIXED_SORTED); this.comp = comp; } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (copie de travail) @@ -25,6 +25,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; /** * Abstract API for per-document stored primitive values of type byte[] @@ -77,14 +78,16 @@ * the {@link Directory} to create the files from. * @param bytesUsed * a byte-usage tracking reference - * @param fasterButMoreRam Whether the space used for packed ints should be rounded up for higher lookup performance. - * Currently this parameter only applies for types {@link Type#BYTES_VAR_SORTED} - * and {@link Type#BYTES_FIXED_SORTED}. + * @param acceptableOverheadRatio + * how to trade space for speed. This option is only applicable for + * docvalues of type {@link Type#BYTES_FIXED_SORTED} and + * {@link Type#BYTES_VAR_SORTED}. * @return a new {@link Writer} instance for the given {@link Type} * @throws IOException + * @see PackedInts#getReader(org.apache.lucene.store.DataInput, float) */ public static DocValuesConsumer create(Type type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { if (comp == null) { comp = BytesRef.getUTF8SortedAsUnicodeComparator(); } @@ -101,22 +104,22 @@ return Floats.getWriter(directory, id, bytesUsed, context, type); case BYTES_FIXED_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_FIXED_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_FIXED_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); default: throw new IllegalArgumentException("Unknown Values: " + type); } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java (copie de travail) @@ -31,6 +31,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; /** * Abstract base class for PerDocConsumer implementations @@ -41,7 +42,7 @@ protected final String segmentName; private final Counter bytesUsed; protected final IOContext context; - private final boolean fasterButMoreRam; + private final float acceptableOverheadRatio; /** * Filename extension for index files @@ -57,20 +58,22 @@ * @param state The state to initiate a {@link PerDocConsumer} instance */ protected DocValuesWriterBase(PerDocWriteState state) { - this(state, true); + this(state, PackedInts.FAST); } /** * @param state The state to initiate a {@link PerDocConsumer} instance - * @param fasterButMoreRam whether packed ints for docvalues should be optimized for speed by rounding up the bytes - * used for a value to either 8, 16, 32 or 64 bytes. This option is only applicable for - * docvalues of type {@link Type#BYTES_FIXED_SORTED} and {@link Type#BYTES_VAR_SORTED}. + * @param acceptableOverheadRatio + * how to trade space for speed. This option is only applicable for + * docvalues of type {@link Type#BYTES_FIXED_SORTED} and + * {@link Type#BYTES_VAR_SORTED}. + * @see PackedInts#getReader(org.apache.lucene.store.DataInput, float) */ - protected DocValuesWriterBase(PerDocWriteState state, boolean fasterButMoreRam) { + protected DocValuesWriterBase(PerDocWriteState state, float acceptableOverheadRatio) { this.segmentName = state.segmentName; this.bytesUsed = state.bytesUsed; this.context = state.context; - this.fasterButMoreRam = fasterButMoreRam; + this.acceptableOverheadRatio = acceptableOverheadRatio; } protected abstract Directory getDirectory() throws IOException; @@ -83,7 +86,7 @@ public DocValuesConsumer addValuesField(Type valueType, FieldInfo field) throws IOException { return Writer.create(valueType, PerDocProducerBase.docValuesId(segmentName, field.number), - getDirectory(), getComparator(), bytesUsed, context, fasterButMoreRam); + getDirectory(), getComparator(), bytesUsed, context, acceptableOverheadRatio); } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedDerefBytesImpl.java (copie de travail) @@ -102,7 +102,7 @@ super(datIn, idxIn, new PagedBytes(PAGED_BYTES_BITS), size * numValues, Type.BYTES_FIXED_DEREF); this.size = size; - addresses = PackedInts.getReader(idxIn); + addresses = PackedInts.getReader(idxIn, PackedInts.DEFAULT); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/PackedIntValues.java (copie de travail) @@ -231,7 +231,7 @@ super(Type.VAR_INTS); minValue = dataIn.readLong(); defaultValue = dataIn.readLong(); - values = direct ? PackedInts.getDirectReader(dataIn) : PackedInts.getReader(dataIn); + values = direct ? PackedInts.getDirectReader(dataIn) : PackedInts.getReader(dataIn, PackedInts.DEFAULT); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (copie de travail) @@ -58,8 +58,8 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_VAR_SORTED); + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_VAR_SORTED); this.comp = comp; size = 0; } @@ -125,7 +125,7 @@ // total bytes of data idxOut.writeLong(maxBytes); PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1, - bitsRequired(maxBytes)); + PackedInts.bitsRequired(maxBytes)); // first dump bytes data, recording index & write offset as // we go final BytesRef spare = new BytesRef(); Index: lucene/core/src/java/org/apache/lucene/codecs/FixedGapTermsIndexReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/FixedGapTermsIndexReader.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/FixedGapTermsIndexReader.java (copie de travail) @@ -301,11 +301,11 @@ termBytes.copy(clone, numTermBytes); // records offsets into main terms dict file - termsDictOffsets = PackedInts.getReader(clone); + termsDictOffsets = PackedInts.getReader(clone, PackedInts.DEFAULT); assert termsDictOffsets.size() == numIndexTerms; // records offsets into byte[] term data - termOffsets = PackedInts.getReader(clone); + termOffsets = PackedInts.getReader(clone, PackedInts.DEFAULT); assert termOffsets.size() == 1+numIndexTerms; } finally { clone.close(); @@ -328,8 +328,8 @@ // we'd have to try @ fewer bits and then grow // if we overflowed it. - PackedInts.Mutable termsDictOffsetsM = PackedInts.getMutable(this.numIndexTerms, termsDictOffsetsIter.getBitsPerValue()); - PackedInts.Mutable termOffsetsM = PackedInts.getMutable(this.numIndexTerms+1, termOffsetsIter.getBitsPerValue()); + PackedInts.Mutable termsDictOffsetsM = PackedInts.getMutable(this.numIndexTerms, termsDictOffsetsIter.getBitsPerValue(), PackedInts.DEFAULT); + PackedInts.Mutable termOffsetsM = PackedInts.getMutable(this.numIndexTerms+1, termOffsetsIter.getBitsPerValue(), PackedInts.DEFAULT); termsDictOffsets = termsDictOffsetsM; termOffsets = termOffsetsM; Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (copie de travail) @@ -74,7 +74,7 @@ PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput(); final int bitEstimate = 1+MathUtil.log(tiiFileLength, 2); - GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, false); + GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, PackedInts.DEFAULT); String currentField = null; List fieldStrs = new ArrayList(); Index: lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (copie de travail) @@ -1071,14 +1071,12 @@ } } - private static boolean DEFAULT_FASTER_BUT_MORE_RAM = true; - public DocTermsIndex getTermsIndex(AtomicReader reader, String field) throws IOException { - return getTermsIndex(reader, field, DEFAULT_FASTER_BUT_MORE_RAM); + return getTermsIndex(reader, field, PackedInts.DEFAULT); } - public DocTermsIndex getTermsIndex(AtomicReader reader, String field, boolean fasterButMoreRAM) throws IOException { - return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)), false); + public DocTermsIndex getTermsIndex(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException { + return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new Entry(field, acceptableOverheadRatio), false); } static class DocTermsIndexCache extends Cache { @@ -1092,7 +1090,7 @@ Terms terms = reader.terms(entryKey.field); - final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue(); + final float acceptableOverheadRatio = ((Float) entryKey.custom).floatValue(); final PagedBytes bytes = new PagedBytes(15); @@ -1142,8 +1140,8 @@ startNumUniqueTerms = 1; } - GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1+startNumUniqueTerms, fasterButMoreRAM); - final GrowableWriter docToTermOrd = new GrowableWriter(startTermsBPV, maxDoc, fasterButMoreRAM); + GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1+startNumUniqueTerms, acceptableOverheadRatio); + final GrowableWriter docToTermOrd = new GrowableWriter(startTermsBPV, maxDoc, acceptableOverheadRatio); // 0 is reserved for "unset" bytes.copyUsingLengthPrefix(new BytesRef()); @@ -1219,11 +1217,11 @@ // TODO: this if DocTermsIndex was already created, we // should share it... public DocTerms getTerms(AtomicReader reader, String field) throws IOException { - return getTerms(reader, field, DEFAULT_FASTER_BUT_MORE_RAM); + return getTerms(reader, field, PackedInts.DEFAULT); } - public DocTerms getTerms(AtomicReader reader, String field, boolean fasterButMoreRAM) throws IOException { - return (DocTerms) caches.get(DocTerms.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)), false); + public DocTerms getTerms(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException { + return (DocTerms) caches.get(DocTerms.class).get(reader, new Entry(field, acceptableOverheadRatio), false); } static final class DocTermsCache extends Cache { @@ -1237,7 +1235,7 @@ Terms terms = reader.terms(entryKey.field); - final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue(); + final float acceptableOverheadRatio = ((Float) entryKey.custom).floatValue(); final int termCountHardLimit = reader.maxDoc(); @@ -1268,7 +1266,7 @@ startBPV = 1; } - final GrowableWriter docToOffset = new GrowableWriter(startBPV, reader.maxDoc(), fasterButMoreRAM); + final GrowableWriter docToOffset = new GrowableWriter(startBPV, reader.maxDoc(), acceptableOverheadRatio); // pointer==0 means not set bytes.copyUsingLengthPrefix(new BytesRef()); Index: lucene/core/src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCache.java (révision 1340914) +++ lucene/core/src/java/org/apache/lucene/search/FieldCache.java (copie de travail) @@ -494,7 +494,7 @@ * faster lookups (default is "true"). Note that the * first call for a given reader and field "wins", * subsequent calls will share the same cache entry. */ - public DocTerms getTerms (AtomicReader reader, String field, boolean fasterButMoreRAM) + public DocTerms getTerms (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException; /** Returned by {@link #getTermsIndex} */ @@ -571,7 +571,7 @@ * faster lookups (default is "true"). Note that the * first call for a given reader and field "wins", * subsequent calls will share the same cache entry. */ - public DocTermsIndex getTermsIndex (AtomicReader reader, String field, boolean fasterButMoreRAM) + public DocTermsIndex getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException; /** Index: lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (révision 1340914) +++ lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (copie de travail) @@ -18,7 +18,9 @@ */ import org.apache.lucene.store.*; +import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.packed.PackedInts.Mutable; import java.util.ArrayList; import java.util.List; @@ -72,7 +74,7 @@ out.close(); {// test reader IndexInput in = d.openInput("out.bin", newIOContext(random())); - PackedInts.Reader r = PackedInts.getReader(in); + PackedInts.Reader r = PackedInts.getReader(in, PackedInts.DEFAULT); assertEquals(fp, in.getFilePointer()); for(int i=0;i