NOTE: this class is meant only to be used internally + * by Lucene; it's only public so it can be shared across + * packages. This means the API is freely subject to + * change, and, the class could be removed entirely, in any + * Lucene release. Use directly at your own risk! + */ + +// nocommit +// - do we need int/long variants (for perf)? or long +// only suffices? +// - what native type is best perf? long/int/short/byte? + +public class PackedInts { + + private final static String CODEC_NAME = "PackedInts"; + private final static int VERSION_START = 0; + private final static int VERSION_CURRENT = 0; + + /** + * The priority for selecting the Reader and Writer implementation. + *
+ * packed: Pack the bits right after each other.
+ * aligned: Pack bits so that no values cross block boundaries.
+ * auto: Guesstimate the best implementation.
+ *
+ * Note: When a more efficient structure (in terms of memory as well as speed) + * can be substituted without penalty, this will be done. Example: + * Asking for packed with 3 bits/value will return packed32 or packed64, while + * asking for packed with 4 bits/value will return aligned32 or aligned64. + * Asking for aligned with 7 bits/value and block preferences bit32 will + * return directByte, as this amount of space used by an aligned32 with 7 + * bits/value is the same as directByte, while directByte is less processor- + * intensive. + *
+ * Note: 63 bits/value will always be mapped to a directLong, due to the + * problem of stating maxValues > 2^63-1. + */ + public enum PRIORITY {packed, aligned, auto} + + /** + * The preference for the underlying blocks for packed or aligned structures. + * Using 64bit blocks (longs) on a 32bit machine is slower than using 32bit + * blocks (ints). + */ + public enum BLOCK_PREFERENCE {bit32(32), bit64(64); + private int bits; + BLOCK_PREFERENCE(int bits) { + this.bits = bits; + } + public int getBits() { + return bits; + } + } + + /** + * The specific implementation derived from bits/value, PRIORITY and + * BLOCK_PREFERENCE. + */ + private enum IMPLEMENTATION {packed32, packed64, aligned32, aligned64, + directByte, directShort, directInt, directLong} + + /** + * + */ + enum PERSISTENCE {packed, aligned32, aligned64} + + /** + * Derives the optimal IMPLEMENTATION based on the given preferences. + * @param bitsPerValue the number of bits available for any given value. + * @param priority memory/speed trade-off. + * @param block the expected architecture for the system that will + * use the Reader-part of the structure. + * @return the implementation to use. + */ + private static IMPLEMENTATION getImplementation( + int bitsPerValue, PRIORITY priority, BLOCK_PREFERENCE block) { + switch (priority) { + case aligned: { + if (block == BLOCK_PREFERENCE.bit32) { + if (bitsPerValue == 7 || bitsPerValue >= 11) { + bitsPerValue = getNextFixedSize(bitsPerValue); // Align to byte, short, int or long + } + } else { + if ((bitsPerValue >= 13 && bitsPerValue <= 15) || + (bitsPerValue >= 22)) { + bitsPerValue = getNextFixedSize(bitsPerValue); // Align to short, int or long + } + } + } + } + if (priority == PRIORITY.auto) { + if (bitsPerValue > 58 || (bitsPerValue < 32 && bitsPerValue > 29)) { // 10% space-waste is ok + bitsPerValue = getNextFixedSize(bitsPerValue); + } + } + + switch (bitsPerValue) { // The safe choices + case 8: return IMPLEMENTATION.directByte; + case 16: return IMPLEMENTATION.directShort; + case 31: + case 32: return IMPLEMENTATION.directInt; + case 63: + case 64: return IMPLEMENTATION.directLong; + } + + if (priority == PRIORITY.aligned || + bitsPerValue == 1 || bitsPerValue == 2 || bitsPerValue == 4) { + if (block == BLOCK_PREFERENCE.bit32 && bitsPerValue < 32) { + return IMPLEMENTATION.aligned32; + } + return IMPLEMENTATION.aligned64; + } + return block == BLOCK_PREFERENCE.bit32 && bitsPerValue < 32 ? + IMPLEMENTATION.packed32 : IMPLEMENTATION.packed64; + } + + /** + * Derives the optimal IMPLEMENTATION based on the given preferences. + * Used for selecting the correct implementation from persistent data. + * @param persistence the format of the existing data. + * @param bitsPerValue the number of bits available for any given value. + * @param block the expected architecture for the system that will + * use the Reader-part of the structure. + * @return the implementation to use. + */ + private static IMPLEMENTATION getImplementation( + PERSISTENCE persistence, int bitsPerValue, BLOCK_PREFERENCE block) { + switch (bitsPerValue) { // The safe choices + case 1: + case 2: + case 4: { + if (block == BLOCK_PREFERENCE.bit32) { + return IMPLEMENTATION.aligned32; + } + return IMPLEMENTATION.aligned64; + } + case 8: return IMPLEMENTATION.directByte; + case 16: return IMPLEMENTATION.directShort; + case 31: + case 32: return IMPLEMENTATION.directInt; + case 63: + case 64: return IMPLEMENTATION.directLong; + } + if (persistence == PERSISTENCE.aligned32) { + return IMPLEMENTATION.aligned32; + } else if (persistence == PERSISTENCE.aligned64) { + return IMPLEMENTATION.aligned64; + } + return block == BLOCK_PREFERENCE.bit32 && bitsPerValue < 32 ? + IMPLEMENTATION.packed32 : IMPLEMENTATION.packed64; + } + + /** Returns how many bits are required to hold values up + * to and including maxValue */ + public static int bitsRequired(long maxValue) { + // Very high long values does not translate well to double, so we do an + // explicit check for the edge cases + if (maxValue > 0x3FFFFFFFFFFFFFFFL) { + return 63; + } if (maxValue > 0x1FFFFFFFFFFFFFFFL) { + return 62; + } + return (int) Math.ceil(Math.log(1+maxValue)/Math.log(2.0)); + } + + /** + * Calculates the maximum unsigned long that can be expressed with the given + * number of bits. + * @param bitsPerValue the number of bits available for any given value. + * @return the maximum value for the given bits. + */ + public static long maxValue(int bitsPerValue) { + return bitsPerValue == 64 ? Long.MAX_VALUE : ~(~0L << bitsPerValue); + } + + private static int getNextFixedSize(int bits) { + if (bits <= 8) { + return 8; + } else if (bits <= 16) { + return 16; + } else if (bits <= 32) { + return 32; + } else { + return 64; + } + } + + /** Write-once */ + public static abstract class Writer { + protected final IndexOutput out; + protected final int bitsPerValue; + protected final int valueCount; + + protected Writer(IndexOutput out, int valueCount, + int bitsPerValue, PERSISTENCE persistence) + throws IOException { + assert bitsPerValue <= 64; + + this.out = out; + this.valueCount = valueCount; + this.bitsPerValue = bitsPerValue; + CodecUtil.writeHeader(out, CODEC_NAME, VERSION_START); + out.writeString(persistence.toString()); + out.writeVInt(bitsPerValue); + out.writeVInt(valueCount); +// System.out.println("Writer PERSISTENCE: " + persistence + " bitsPerValue: " + bitsPerValue); + } + + public abstract void add(long v) throws IOException; + public abstract void finish() throws IOException; + } + + public static Writer getWriter( + IndexOutput out, int valueCount, int bitsPerValue, + PRIORITY priority, BLOCK_PREFERENCE block) throws IOException { + IMPLEMENTATION implementation = getImplementation( + bitsPerValue, priority, block); + switch (implementation) { + case packed32: + case packed64: + return new PackedWriter(out, valueCount, bitsPerValue); + case directByte: + return new PackedWriter(out, valueCount, 8); + case directShort: + return new PackedWriter(out, valueCount, 16); + case directInt: + return new PackedWriter(out, valueCount, 32); + case directLong: + return new PackedWriter(out, valueCount, 64); + case aligned32: + return new PackedAlignedWriter( + out, valueCount, bitsPerValue, BLOCK_PREFERENCE.bit32); + case aligned64: + return new PackedAlignedWriter( + out, valueCount, bitsPerValue, BLOCK_PREFERENCE.bit64); + default: throw new UnsupportedOperationException( + implementation + " is not implemented yet"); + } + } + + public static Mutable getMutable( + int valueCount, int bitsPerValue, + PRIORITY priority, BLOCK_PREFERENCE block) throws IOException { + IMPLEMENTATION implementation = getImplementation( + bitsPerValue, priority, block); + switch (implementation) { + case packed32: return new Packed32(valueCount, bitsPerValue); + case packed64: return new Packed64(valueCount, bitsPerValue); + case directByte: return new PackedDirectByte(valueCount); + case directShort: return new PackedDirectShort(valueCount); + case directInt: return new PackedDirectInt(valueCount); + case directLong: return new PackedDirectLong(valueCount); + case aligned32: return new PackedAligned32(valueCount, bitsPerValue); + case aligned64: return new PackedAligned64(valueCount, bitsPerValue); + default: throw new UnsupportedOperationException( + implementation + " is not implemented yet"); + } + } + + public static interface Reader extends ConsumesRAM { + /** + * @param index the position of the wanted value. + * @return the value at the stated index. + */ + long get(int index); + + /** + * @return the number of bits used to store any given value. + * Note: This does not imply that memory usage is + * {@code bitsPerValue * #values} as implementations are free to + * use non-space-optimal packing of bits. + */ + int getBitsPerValue(); + + /** + * @return the number of values. + */ + int size(); + } + + /** + * A packed integer array that can be modified. + */ + public static interface Mutable extends Reader { + /** + * Set the value at the given index in the array. + * @param index where the value should be positioned. + * @param value a value conforming to the constraints set by the array. + */ + void set(int index, long value); + + /** + * Sets all values to 0. + */ + void clear(); + } + + public static abstract class ReaderImpl implements Reader { + protected final int bitsPerValue; + protected final int valueCount; + + protected ReaderImpl(int valueCount, int bitsPerValue) { + this.bitsPerValue = bitsPerValue; + this.valueCount = valueCount; + } + + public int getBitsPerValue() { + return bitsPerValue; + } + + public int size() { + return valueCount; + } + + public long getMaxValue() { // Convenience method + return maxValue(bitsPerValue); + } + } + + public static Reader getReader(IndexInput in) throws IOException { + return getReader(in, Constants.JRE_IS_64BIT ? + BLOCK_PREFERENCE.bit64 : BLOCK_PREFERENCE.bit32); + } + public static Reader getReader(IndexInput in, BLOCK_PREFERENCE block) + throws IOException { + CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START); + String pStr = in.readString(); + PERSISTENCE persistence = PERSISTENCE.valueOf(pStr); + final int bitsPerValue = in.readVInt(); + final int valueCount = in.readVInt(); +// final long maxValue = in.readVLong(); + + IMPLEMENTATION implementation = + getImplementation(persistence, bitsPerValue, block); +// System.out.println("getReader PERSISTENCE: " + persistence + " bitsPerValue: " + bitsPerValue + " IMPLEMENTATION: " + implementation); + switch (implementation) { + case packed32: return new Packed32(in, valueCount, bitsPerValue); + case packed64: return new Packed64(in, valueCount, bitsPerValue); + case aligned32: return new PackedAligned32(in, valueCount, bitsPerValue); + case aligned64: return new PackedAligned64(in, valueCount, bitsPerValue); + case directByte: return new PackedDirectByte(in, valueCount); + case directShort: return new PackedDirectShort(in, valueCount); + case directInt: return new PackedDirectInt(in, valueCount); + case directLong: return new PackedDirectLong(in, valueCount); + default: throw new UnsupportedOperationException("Not implemented yet"); + } + + // TODO an mmap reader as well? + } +} Index: src/java/org/apache/lucene/util/CodecUtil.java =================================================================== --- src/java/org/apache/lucene/util/CodecUtil.java Fri Jan 22 12:58:35 CET 2010 +++ src/java/org/apache/lucene/util/CodecUtil.java Fri Jan 22 12:58:35 CET 2010 @@ -0,0 +1,72 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/** + *
NOTE: this class is meant only to be used internally
+ * by Lucene; it's only public so it can be shared across
+ * packages. This means the API is freely subject to
+ * change, and, the class could be removed entirely, in any
+ * Lucene release. Use directly at your own risk!
+ */
+
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.index.CorruptIndexException;
+
+import java.io.IOException;
+
+public final class CodecUtil {
+ private final static int CODEC_MAGIC = 0x3fd76c17;
+
+ public static void writeHeader(IndexOutput out, String codec, int version)
+ throws IOException {
+ final long start = out.getFilePointer();
+ out.writeInt(CODEC_MAGIC);
+ out.writeString(codec);
+ out.writeInt(version);
+
+ // We require this so we can easily pre-compute header length
+ if (out.getFilePointer()-start != codec.length()+9) {
+ throw new IllegalArgumentException("codec must be simple ASCII, less than 128 characters in length [got " + codec + "]");
+ }
+ }
+
+ public static int headerLength(String codec) {
+ return 9+codec.length();
+ }
+
+ public static int checkHeader(IndexInput in, String codec, int maxVersion)
+ throws IOException {
+ final int actualHeader = in.readInt();
+ if (actualHeader != CODEC_MAGIC) {
+ throw new CorruptIndexException("codec header mismatch: actual header=" + actualHeader + " vs expected header=" + CODEC_MAGIC);
+ }
+ final String actualCodec = in.readString();
+ if (!actualCodec.equals(codec)) {
+ throw new CorruptIndexException("codec mismatch: actual codec=" + actualCodec + " vs expected codec=" + codec);
+ }
+ final int actualVersion = in.readInt();
+ if (actualVersion > maxVersion) {
+ throw new CorruptIndexException("version " + actualVersion + " is too new (expected <= version " + maxVersion + ")");
+ }
+
+ return actualVersion;
+ }
+}
Index: src/java/org/apache/lucene/util/packed/PackedDirectLong.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedDirectLong.java Fri Feb 12 01:29:57 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedDirectLong.java Fri Feb 12 01:29:57 CET 2010
@@ -0,0 +1,79 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Direct wrapping of 32 bit values to a backing array of ints.
+ */
+public class PackedDirectLong extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ private long[] blocks;
+ private static final int BITS_PER_VALUE = 64;
+
+ public PackedDirectLong(int valueCount) {
+ super(valueCount, BITS_PER_VALUE);
+ blocks = new long[valueCount];
+ }
+
+ public PackedDirectLong(IndexInput in, int valueCount) throws IOException {
+ super(valueCount, BITS_PER_VALUE);
+ long[] blocks = new long[valueCount];
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the structure.
+ * @param blocks used as the internal backing array.
+ */
+ public PackedDirectLong(long[] blocks) {
+ super(blocks.length, BITS_PER_VALUE);
+ this.blocks = blocks;
+ }
+
+ public long get(final int index) {
+ return blocks[index];
+ }
+
+ public void set(final int index, final long value) {
+ blocks[index] = value;
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
+ blocks.length * RamUsageEstimator.NUM_BYTES_LONG;
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, 0L);
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/packed/Packed32.java
===================================================================
--- src/java/org/apache/lucene/util/packed/Packed32.java Fri Feb 12 01:39:36 CET 2010
+++ src/java/org/apache/lucene/util/packed/Packed32.java Fri Feb 12 01:39:36 CET 2010
@@ -0,0 +1,218 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Space optimized random access capable array of values with a fixed number of
+ * bits. The maximum number of bits/value is 31. Use {@link Packed64} for higher
+ * numbers.
+ *
+ * The implementation strives to avoid conditionals and expensive operations,
+ * sacrificing code clarity to achieve better performance.
+ */
+public class Packed32 extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ static final int BLOCK_SIZE = 32; // 32 = int, 64 = long
+ static final int BLOCK_BITS = 5; // The #bits representing BLOCK_SIZE
+ static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE
+
+ private static final int ENTRY_SIZE = BLOCK_SIZE + 1;
+ private static final int FAC_BITPOS = 3;
+
+ /*
+ * In order to make an efficient value-getter, conditionals should be
+ * avoided. A value can be positioned inside of a block, requiring shifting
+ * left or right or it can span two blocks, requiring a left-shift on the
+ * first block and a right-shift on the right block.
+ *
+ * By always shifting the first block both left and right, we get exactly
+ * the right bits. By always shifting the second block right and applying
+ * a mask, we get the right bits there. After that, we | the two bitsets.
+ */
+ private static final int[][] SHIFTS =
+ new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
+ private static final int[][] MASKS = new int[ENTRY_SIZE][ENTRY_SIZE];
+
+ static { // Generate shifts
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ int[] currentShifts = SHIFTS[elementBits];
+ int base = bitPos * FAC_BITPOS;
+ currentShifts[base ] = bitPos;
+ currentShifts[base + 1] = BLOCK_SIZE - elementBits;
+ if (bitPos <= BLOCK_SIZE - elementBits) { // Single block
+ currentShifts[base + 2] = 0;
+ MASKS[elementBits][bitPos] = 0;
+ } else { // Two blocks
+ int rBits = elementBits - (BLOCK_SIZE - bitPos);
+ currentShifts[base + 2] = BLOCK_SIZE - rBits;
+ MASKS[elementBits][bitPos] = ~(~0 << rBits);
+ }
+ }
+ }
+ }
+
+ /*
+ * The setter requires more masking than the getter.
+ */
+ private static final int[][] WRITE_MASKS =
+ new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
+ static {
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ int elementPosMask = ~(~0 << elementBits);
+ int[] currentShifts = SHIFTS[elementBits];
+ int[] currentMasks = WRITE_MASKS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ int base = bitPos * FAC_BITPOS;
+ currentMasks[base ] =~((elementPosMask
+ << currentShifts[base + 1])
+ >>> currentShifts[base]);
+ currentMasks[base+1] = ~(elementPosMask
+ << currentShifts[base + 2]);
+ currentMasks[base+2] = currentShifts[base + 2] == 0 ? 0 : ~0;
+ }
+ }
+ }
+
+ /* The bits */
+ private int[] blocks;
+
+ // Cached calculations
+ private int maxPos; // blocks.length * BLOCK_SIZE / bitsPerValue - 1
+ private int[] shifts; // The shifts for the current bitsPerValue
+ private int[] readMasks;
+ private int[] writeMasks;
+
+ /**
+ * Creates an array with the internal structures adjusted for the given
+ * limits and initialized to 0.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ * Note: bitsPerValue >32 is not supported by this implementation.
+ */
+ public Packed32(int valueCount, int bitsPerValue) {
+ this(new int[(int)(((long)valueCount) * bitsPerValue / BLOCK_SIZE + 2)],
+ valueCount, bitsPerValue);
+ }
+
+ /**
+ * Creates an array with content retrieved from the given IndexInput.
+ * @param in an IndexInput, positioned at the start of Packed64-content.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ * @throws java.io.IOException if the values for the backing array could not
+ * be retrieved.
+ */
+ public Packed32(IndexInput in, int valueCount, int bitsPerValue)
+ throws IOException {
+ super(valueCount, bitsPerValue);
+ int size = size(bitsPerValue, valueCount);
+ blocks = new int[size+1]; // +1 due to non-conditional tricks
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the Packed32-structure.
+ * @param blocks used as the internal backing array.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ * Note: bitsPerValue >32 is not supported by this implementation.
+ */
+ public Packed32(int[] blocks, int valueCount, int bitsPerValue) {
+ // TODO: Check that blocks.length is sufficient for holding length values
+ super(valueCount, bitsPerValue);
+ if (bitsPerValue > 31) {
+ throw new IllegalArgumentException(String.format(
+ "This array only supports values of 31 bits or less. The "
+ + "required number of bits was %d. The Packed64 "
+ + "implementation allows values with more than 31 bits",
+ bitsPerValue));
+ }
+ this.blocks = blocks;
+ updateCached();
+ }
+
+ private void updateCached() {
+ readMasks = MASKS[bitsPerValue];
+ maxPos = (int)((((long)blocks.length) * BLOCK_SIZE / bitsPerValue) - 2);
+ shifts = SHIFTS[bitsPerValue];
+ writeMasks = WRITE_MASKS[bitsPerValue];
+ }
+
+ /**
+ * @param index the position of the value.
+ * @return the value at the given index.
+ */
+ public long get(final int index) {
+ final long majorBitPos = index * bitsPerValue;
+ final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
+ final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
+
+ final int base = bitPos * FAC_BITPOS;
+
+ return ((blocks[elementPos] << shifts[base]) >>> shifts[base+1]) |
+ ((blocks[elementPos+1] >>> shifts[base+2])
+ & readMasks[bitPos]);
+ }
+
+ public void set(final int index, final long value) {
+ final int intValue = (int)value;
+ final long majorBitPos = index * bitsPerValue;
+ final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
+ final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
+ final int base = bitPos * FAC_BITPOS;
+
+ blocks[elementPos ] = (blocks[elementPos ] & writeMasks[base])
+ | (intValue << shifts[base + 1] >>> shifts[base]);
+ blocks[elementPos+1] = (blocks[elementPos+1] & writeMasks[base+1])
+ | ((intValue << shifts[base + 2])
+ & writeMasks[base+2]);
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, 0);
+ }
+
+ public String toString() {
+ return "Packed32(bitsPerValue=" + bitsPerValue + ", maxPos=" + maxPos
+ + ", elements.length=" + blocks.length + ")";
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ + blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ }
+}
Index: src/java/org/apache/lucene/store/IndexInput.java
===================================================================
--- src/java/org/apache/lucene/store/IndexInput.java (revision 895342)
+++ src/java/org/apache/lucene/store/IndexInput.java Fri Jan 22 12:58:35 CET 2010
@@ -64,6 +64,13 @@
readBytes(b, offset, len);
}
+ /** Reads two bytes and returns a short.
+ * @see IndexOutput#writeInt(int)
+ */
+ public short readShort() throws IOException {
+ return (short) (((readByte() & 0xFF) << 8) | (readByte() & 0xFF));
+ }
+
/** Reads four bytes and returns an int.
* @see IndexOutput#writeInt(int)
*/
Index: src/java/org/apache/lucene/util/packed/PackedAligned32.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedAligned32.java Fri Feb 12 02:19:44 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedAligned32.java Fri Feb 12 02:19:44 CET 2010
@@ -0,0 +1,185 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Medium space and speed trade off. No values crosses block boundaries.
+ * The maximum number of bits/value is 32.
+ * Use {@link org.apache.lucene.util.packed.PackedAligned64} for higher numbers.
+ *
+ * The implementation strives to avoid conditionals and expensive operations,
+ * sacrificing code clarity to achieve better performance.
+ *
+ * Space is optimally used within the boundaries of alignment, e.g.
+ * 7 bits/value fits 4 values/block for 32 bit and 7 values/block for 64 bit.
+ * Bits are packed left-aligned to be bit pattern compatible with other bit
+ * array implementations where possible.
+ */
+public class PackedAligned32 extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ static final int BLOCK_SIZE = 32; // 32 = int, 64 = long
+
+ private static final int ENTRY_SIZE = BLOCK_SIZE + 1;
+
+ /*
+ * A value is always positioned inside a single block, requiring a
+ * shift right to position the bits and a mask to extract them.
+ */
+ private static final int[][] SHIFTS = new int[ENTRY_SIZE][ENTRY_SIZE];
+ private static final int[] READ_MASKS = new int[ENTRY_SIZE];
+
+ static { // Generate shifts
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ int[] currentShifts = SHIFTS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ currentShifts[bitPos] = BLOCK_SIZE + bitPos - elementBits;
+ READ_MASKS[elementBits] = ~(~0 << elementBits);
+ }
+ }
+ }
+
+ /*
+ * Setting a value requires clearing the destination bits with a mask, then
+ * shifting the value to the left and or'ing the two numbers.
+ */
+ private static final int[][] WRITE_MASKS = new int[ENTRY_SIZE][ENTRY_SIZE];
+ static {
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ int elementPosMask = ~(~0 << elementBits);
+ int[] currentShifts = SHIFTS[elementBits];
+ int[] currentMasks = WRITE_MASKS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ currentMasks[bitPos] = ~(elementPosMask
+ << currentShifts[bitPos]);
+ }
+ }
+ }
+
+ /* The bits */
+ private int[] blocks;
+
+ /* Cached values */
+ private int valuesPerBlock;
+ private int[] shifts;
+ private int readMask;
+ private int[] writeMasks;
+
+ /**
+ * Creates an array with the internal structures adjusted for the given
+ * limits and initialized to 0.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public PackedAligned32(int valueCount, int bitsPerValue) {
+ super(valueCount, bitsPerValue);
+ if (bitsPerValue > 32) {
+ throw new IllegalArgumentException(String.format(
+ "This array only supports values of 32 bits or less. The "
+ + "required number of bits was %d. The PackedAligned64 "
+ + "implementation allows values with more than 32 bits",
+ bitsPerValue));
+ }
+ blocks = new int[size(valueCount, bitsPerValue)];
+ updateCached();
+ }
+
+ private static int size(int valueCount, int bitsPerValue) {
+ int valuesPerBlock = BLOCK_SIZE / bitsPerValue;
+ return valueCount / valuesPerBlock + 2;
+ }
+
+ /**
+ * Creates an array with content retrieved from the given IndexInput.
+ * @param in an IndexInput, positioned at the start of Packed64-content.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ * @throws java.io.IOException if the values for the backing array could not
+ * be retrieved.
+ */
+ public PackedAligned32(IndexInput in, int valueCount, int bitsPerValue)
+ throws IOException {
+ super(valueCount, bitsPerValue);
+ int size = size(valueCount, bitsPerValue);
+ blocks = new int[size];
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the Packed32-structure.
+ * @param blocks used as the internal backing array.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public PackedAligned32(int[] blocks, int valueCount, int bitsPerValue) {
+ // TODO: Check that blocks.length is sufficient for holding length values
+ super(valueCount, bitsPerValue);
+ this.blocks = blocks;
+ updateCached();
+ }
+
+ private void updateCached() {
+ valuesPerBlock = BLOCK_SIZE / bitsPerValue;
+ shifts = SHIFTS[bitsPerValue];
+ readMask = READ_MASKS[bitsPerValue];
+ writeMasks = WRITE_MASKS[bitsPerValue];
+ }
+
+ /**
+ * @param index the position of the value.
+ * @return the value at the given index.
+ */
+ public long get(final int index) {
+ final int blockPos = index / valuesPerBlock;
+ final int bitPos = (index - (blockPos * valuesPerBlock)) * bitsPerValue;
+
+ return (blocks[blockPos] >>> shifts[bitPos]) & readMask;
+ }
+
+ public void set(final int index, final long value) {
+ final int intValue = (int)value;
+
+ final int blockPos = index / valuesPerBlock;
+ final int bitPos = (index - (blockPos * valuesPerBlock)) * bitsPerValue;
+
+ blocks[blockPos] = (blocks[blockPos] & writeMasks[bitPos])
+ | (intValue << shifts[bitPos]);
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, 0);
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ + blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ }
+
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/RamUsageEstimator.java
===================================================================
--- src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 901710)
+++ src/java/org/apache/lucene/util/RamUsageEstimator.java Fri Jan 22 13:01:30 CET 2010
@@ -35,6 +35,16 @@
* estimate is complete.
*/
public final class RamUsageEstimator {
+
+ public static int NUM_BYTES_SHORT = 2;
+ public static int NUM_BYTES_INT = 4;
+ public static int NUM_BYTES_LONG = 8;
+ public static int NUM_BYTES_FLOAT = 4;
+ public static int NUM_BYTES_DOUBLE = 8;
+ public static int NUM_BYTES_OBJ_HEADER = 8;
+ public static int NUM_BYTES_OBJ_REF = Constants.JRE_IS_64BIT ? 8 : 4;
+ public static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJ_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJ_REF;
+
private MemoryModel memoryModel;
private final Map
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the structure.
+ * @param blocks used as the internal backing array.
+ */
+ public PackedDirectInt(int[] blocks) {
+ super(blocks.length, BITS_PER_VALUE);
+ this.blocks = blocks;
+ }
+
+ public long get(final int index) {
+ return 0xFFFFFFFFL & blocks[index];
+ }
+
+ public void set(final int index, final long value) {
+ blocks[index] = (int)(value & 0xFFFFFFFF);
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
+ blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, 0);
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/packed/TODO
===================================================================
--- src/java/org/apache/lucene/util/packed/TODO Fri Feb 12 03:23:08 CET 2010
+++ src/java/org/apache/lucene/util/packed/TODO Fri Feb 12 03:23:08 CET 2010
@@ -0,0 +1,103 @@
+- Make a better check for persistence validity
+All types of writers and readers needs to be checked
+
+- Make PackedAligned64 persistence work
+There's a bug somewhere. The same bug will probably also be in PackedAligned32.
+
+- Test whether aligned is always faster than packed
+Aligned uses more logic (mainly a division), but packed requests two ints/longs
+from RAM for each request. Maybe we can avoid using aligned at all?
+TestPackedInts.testSpeed() does performance testing. This should be done on
+different hardware.
+
+- Ensure that writers align to 64 bit and that they append an empty long
+The extra long is needed by the packed-implementation in order to avoid
+conditionals. Right now is could be spared as we always load into memory and
+could just allocate an extra long, but a switch to mem-mapping or such would
+require the persistent format to contain the long.
+
+ - Determine whether it should be possible to request mutable arrays or not
+Keeping it possible to have mutable arrays opens up for re-use of the
+implementations, but it also makes the implementations heavier to maintain
+due to the added set-method.
+
+- Determine how to request a Writer
+Using an aligned64-structure on a 32 bit machine comes with a heavy
+speed-penalty, due to the Reader having to request a long for each get.
+The overhead is so large that it would be better to use packed (backed by int)
+instead. This means - unfortunately - that the Writer needs to guess the
+architecture that the Reader will be used on.
+Using a factory can help choosing the implementation: Requesting an aligned
+structure with 7 bits/value is a bad idea as the faster directByte takes the
+same amount of space. On a 32 bit machine that is. In order for the factory
+to auto-promote selections, it needs to know whether 32 bit or 64 bit is
+preferable.
+
+- JavaDocs
+
+
+
+********************************************************************************
+testSpeed result by Toke Eskildsen (te@statsbiblioteket.dk) 2010-02-12
+Java 1.6.0_15-b03, default settings
+Dell Precision M6500: Intel i7 Q 820 @ 1.73GHz, 8 MB level 2 cache,
+ dual-channel PC 1333 RAM
+********************************************************************************
+
+ bitsPerValue valueCount getCount PackedDirectByte PackedDirectShort Packed32 PackedAligned32 PackedDirectInt Packed64 PackedAligned64 PackedDirectLong
+ 1 1000 10000000 167 141 258 242 172 264 242 183
+ 1 1000000 10000000 224 232 266 233 246 262 238 338
+ 1 10000000 10000000 359 469 280 278 508 278 272 551
+ 3 1000 10000000 168 166 265 241 163 262 243 166
+ 3 1000000 10000000 227 226 261 251 239 274 249 330
+ 3 10000000 10000000 406 476 301 304 522 300 308 547
+ 4 1000 10000000 167 168 266 239 164 285 239 169
+ 4 1000000 10000000 228 231 294 274 262 291 269 314
+ 4 10000000 10000000 385 480 308 333 514 331 315 557
+ 7 1000 10000000 172 174 278 248 162 271 238 177
+ 7 1000000 10000000 224 236 289 281 272 278 277 345
+ 7 10000000 10000000 405 473 389 447 516 399 402 553
+ 8 1000 10000000 192 171 268 242 174 291 240 163
+ 8 1000000 10000000 226 232 291 284 286 274 265 314
+ 8 10000000 10000000 381 467 406 428 512 422 419 580
+
+ bitsPerValue valueCount getCount PackedDirectShort Packed32 PackedAligned32 PackedDirectInt Packed64 PackedAligned64 PackedDirectLong
+ 9 1000 10000000 166 274 241 170 261 237 163
+ 9 1000000 10000000 229 299 273 250 284 275 327
+ 9 10000000 10000000 483 443 477 519 438 455 568
+ 15 1000 10000000 170 265 239 174 264 235 162
+ 15 1000000 10000000 232 285 274 240 278 269 339
+ 15 10000000 10000000 473 518 524 523 519 521 550
+ 16 1000 10000000 166 263 236 172 264 235 160
+ 16 1000000 10000000 229 285 278 244 293 272 332
+ 16 10000000 10000000 470 513 517 509 534 529 548
+
+ bitsPerValue valueCount getCount Packed32 PackedAligned32 PackedDirectInt Packed64 PackedAligned64 PackedDirectLong
+ 17 1000 10000000 262 255 177 260 234 160
+ 17 1000000 10000000 290 306 273 304 290 320
+ 17 10000000 10000000 532 572 533 529 556 551
+ 28 1000 10000000 269 256 187 267 238 163
+ 28 1000000 10000000 293 295 253 293 296 312
+ 28 10000000 10000000 542 567 501 548 567 542
+ 31 1000 10000000 260 235 177 266 232 158
+ 31 1000000 10000000 292 294 244 296 297 328
+ 31 10000000 10000000 552 563 516 562 568 548
+
+ bitsPerValue valueCount getCount PackedDirectInt Packed64 PackedAligned64 PackedDirectLong
+ 32 1000 10000000 172 263 241 166
+ 32 1000000 10000000 241 291 297 320
+ 32 10000000 10000000 519 556 573 546
+
+ bitsPerValue valueCount getCount Packed64 PackedAligned64 PackedDirectLong
+ 33 1000 10000000 264 239 159
+ 33 1000000 10000000 293 374 319
+ 33 10000000 10000000 559 595 552
+ 47 1000 10000000 264 242 164
+ 47 1000000 10000000 319 369 322
+ 47 10000000 10000000 577 601 548
+ 49 1000 10000000 261 243 162
+ 49 1000000 10000000 323 413 319
+ 49 10000000 10000000 584 610 551
+ 63 1000 10000000 269 235 161
+ 63 1000000 10000000 396 369 313
+ 63 10000000 10000000 592 596 559
Index: src/java/org/apache/lucene/util/packed/PackedDirectShort.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedDirectShort.java Fri Feb 12 01:29:57 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedDirectShort.java Fri Feb 12 01:29:57 CET 2010
@@ -0,0 +1,86 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Direct wrapping of 16 bit values to a backing array of shorts.
+ */
+public class PackedDirectShort extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ private short[] blocks;
+ private static final int BITS_PER_VALUE = 16;
+
+ public PackedDirectShort(int valueCount) {
+ super(valueCount, BITS_PER_VALUE);
+ blocks = new short[valueCount];
+ }
+
+ public PackedDirectShort(IndexInput in, int valueCount) throws IOException {
+ super(valueCount, BITS_PER_VALUE);
+ short[] blocks = new short[valueCount];
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the structure.
+ * @param blocks used as the internal backing array.
+ */
+ public PackedDirectShort(short[] blocks) {
+ super(blocks.length, BITS_PER_VALUE);
+ this.blocks = blocks;
+ }
+
+ public long get(final int index) {
+ return 0xFFFFL & blocks[index];
+ }
+
+ public void set(final int index, final long value) {
+ blocks[index] = (short)(value & 0xFFFF);
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER +
+ blocks.length * RamUsageEstimator.NUM_BYTES_SHORT;
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, (short)0);
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/packed/PackedAlignedWriter.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedAlignedWriter.java Fri Feb 12 02:59:59 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedAlignedWriter.java Fri Feb 12 02:59:59 CET 2010
@@ -0,0 +1,95 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.IndexOutput;
+
+import java.io.IOException;
+
+// Packs high order byte first, to match
+// IndexOutput.writeInt/Long/Short byte order
+
+/**
+ * Generic writer for block-aligned values: Bits for values are stored so
+ * that block-boundaries are never crossed. For some number of bits, this means
+ * wasted space in the blocks.
+ *
+ * The bits for values are stored left-aligned in the blocks, in order to be
+ * bit-pattern compatible with byte, short, int and long-backed implementations
+ * as well as packed for 1, 2, 4, 8, 16, 32 and 64 bits/value.
+ */
+public class PackedAlignedWriter extends PackedInts.Writer {
+ private final PackedInts.BLOCK_PREFERENCE blockPref;
+ private long pending = 0;
+ private int pendingBitPos = 0;
+ private int written = 0;
+
+ public PackedAlignedWriter(IndexOutput out, int valueCount,
+ int bitsPerValue, PackedInts.BLOCK_PREFERENCE blockPref)
+ throws IOException {
+ super(out, valueCount, bitsPerValue,
+ blockPref == PackedInts.BLOCK_PREFERENCE.bit32 ?
+ PackedInts.PERSISTENCE.aligned32 :
+ PackedInts.PERSISTENCE.aligned64);
+ this.blockPref = blockPref;
+ }
+
+ @Override
+ public void add(long value) throws IOException {
+ // TODO: Consider caching maxValue and bits/block
+ assert value <= PackedInts.maxValue(bitsPerValue) : "value=" + value
+ + " maxValue=" + PackedInts.maxValue(bitsPerValue);
+ assert value >= 0;
+ assert written <= valueCount : "The number of values to write has been " +
+ "exceeded, expected number of values: " + valueCount;
+ pending |= value << (blockPref.getBits() - pendingBitPos - bitsPerValue);
+ pendingBitPos += bitsPerValue;
+ if (pendingBitPos > blockPref.getBits() - bitsPerValue) {
+ flush();
+ }
+ written++;
+ }
+
+ @Override
+ public void finish() throws IOException {
+ assert written == valueCount :
+ valueCount + " values should be added, but only " + written
+ + " has been received";
+ if (pendingBitPos != 0) {
+ flush();
+ }
+ out.writeLong(0L); // Dummy last element to be compatible with packed
+ }
+
+ private void flush() throws IOException {
+ // TODO: Align to 64 bit
+ switch (blockPref) {
+ case bit32: {
+ out.writeInt((int)(pending >>> 32));
+ break;
+ }
+ case bit64: {
+ out.writeLong(pending);
+ break;
+ }
+ }
+ pending = 0;
+ pendingBitPos = 0;
+ }
+
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/packed/PackedDirectByte.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedDirectByte.java Fri Feb 12 02:25:36 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedDirectByte.java Fri Feb 12 02:25:36 CET 2010
@@ -0,0 +1,86 @@
+package org.apache.lucene.util.packed;
+
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.RamUsageEstimator;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Direct wrapping of 8 bit values to a backing array of bytes.
+ */
+public class PackedDirectByte extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ private byte[] blocks;
+ private static final int BITS_PER_VALUE = 8;
+
+ public PackedDirectByte(int valueCount) {
+ super(valueCount, BITS_PER_VALUE);
+ blocks = new byte[valueCount];
+ }
+
+ public PackedDirectByte(IndexInput in, int valueCount)
+ throws IOException {
+ super(valueCount, BITS_PER_VALUE);
+ byte[] blocks = new byte[valueCount];
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the structure.
+ * @param blocks used as the internal backing array.
+ */
+ public PackedDirectByte(byte[] blocks) {
+ super(blocks.length, BITS_PER_VALUE);
+ this.blocks = blocks;
+ }
+
+ public long get(final int index) {
+ return 0xFFL & blocks[index];
+ }
+
+ public void set(final int index, final long value) {
+ blocks[index] = (byte)(value & 0xFF);
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + blocks.length;
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, (byte)0);
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/packed/PackedWriter.java
===================================================================
--- src/java/org/apache/lucene/util/packed/PackedWriter.java Fri Feb 12 00:37:49 CET 2010
+++ src/java/org/apache/lucene/util/packed/PackedWriter.java Fri Feb 12 00:37:49 CET 2010
@@ -0,0 +1,105 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.IndexOutput;
+
+import java.io.IOException;
+
+// Packs high order byte first, to match
+// IndexOutput.writeInt/Long/Short byte order
+
+/**
+ * Generic writer for space-optimal packed values. The resulting bits can be
+ * used directly by Packed32, Packed64 and PackedDirect* and will always be
+ * long-aligned.
+ */
+public class PackedWriter extends PackedInts.Writer {
+ private long pending;
+ private int pendingBitPos;
+
+ // masks[n-1] masks for bottom n bits
+ private final long[] masks;
+
+ // nocommit -- allow minValue too? ie not just minValue==0
+
+ public PackedWriter(IndexOutput out, int valueCount, int bitsPerValue)
+ throws IOException {
+
+ super(out, valueCount, bitsPerValue, PackedInts.PERSISTENCE.packed);
+
+ pendingBitPos = 64;
+ masks = new long[bitsPerValue - 1];
+
+ int v = 1;
+ for (int i = 0; i < bitsPerValue - 1; i++) {
+ v *= 2;
+ masks[i] = v - 1;
+ }
+ }
+
+ /**
+ * Do not call this after finish
+ */
+ @Override
+ public void add(long v) throws IOException {
+ assert v <= PackedInts.maxValue(bitsPerValue) : "v=" + v
+ + " maxValue=" + PackedInts.maxValue(bitsPerValue);
+ assert v >= 0;
+ //System.out.println(" packedw add v=" + v + " pendingBitPos=" + pendingBitPos);
+
+ // TODO
+ if (pendingBitPos >= bitsPerValue) {
+ // not split
+
+ // write-once, so we can |= w/o first masking to 0s
+ pending |= v << (pendingBitPos - bitsPerValue);
+ if (pendingBitPos == bitsPerValue) {
+ // flush
+ out.writeLong(pending);
+ pending = 0;
+ pendingBitPos = 64;
+ } else {
+ pendingBitPos -= bitsPerValue;
+ }
+
+ } else {
+ // split
+
+ // write top pendingBitPos bits of value into bottom bits of pending
+ pending |= (v >> (bitsPerValue - pendingBitPos)) & masks[pendingBitPos - 1];
+ //System.out.println(" part1 (v >> " + (bitsPerValue - pendingBitPos) + ") & " + masks[pendingBitPos-1]);
+
+ // flush
+ out.writeLong(pending);
+
+ // write bottom (bitsPerValue - pendingBitPos) bits of value into top bits of pending
+ pendingBitPos = 64 - bitsPerValue + pendingBitPos;
+ //System.out.println(" part2 v << " + pendingBitPos);
+ pending = (v << pendingBitPos);
+ }
+ }
+
+ @Override
+ public void finish() throws IOException {
+ if (pendingBitPos != 64) {
+ out.writeLong(pending);
+ }
+ out.writeLong(0L); // Dummy to compensate for not using conditionals
+ }
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/util/BytesRef.java
===================================================================
--- src/java/org/apache/lucene/util/BytesRef.java Fri Jan 22 12:58:35 CET 2010
+++ src/java/org/apache/lucene/util/BytesRef.java Fri Jan 22 12:58:35 CET 2010
@@ -0,0 +1,170 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.UnsupportedEncodingException;
+
+// nocommit -- share w/ flex's TermRef
+public class BytesRef {
+
+ public byte[] bytes;
+ public int offset;
+ public int length;
+
+ public abstract static class Comparator {
+ abstract public int compare(BytesRef a, BytesRef b);
+ }
+
+ public BytesRef() {
+ }
+
+ /** Creates bytes ref, wrapping UTF8 bytes from the
+ * provided string. */
+ public BytesRef(String s) {
+ try {
+ bytes = s.getBytes("UTF-8");
+ } catch (UnsupportedEncodingException uee) {
+ throw new RuntimeException(uee);
+ }
+ offset = 0;
+ length = bytes.length;
+ }
+
+ public BytesRef(BytesRef other) {
+ offset = 0;
+ length = other.length;
+ bytes = new byte[other.length];
+ System.arraycopy(other.bytes, other.offset, bytes, 0, length);
+ }
+
+ public boolean bytesEquals(BytesRef other) {
+ if (length == other.length) {
+ int upto = offset;
+ int otherUpto = other.offset;
+ final byte[] otherBytes = other.bytes;
+ for(int i=0;i
+ * The implementation strives to avoid conditionals and expensive operations,
+ * sacrificing code clarity to achieve better performance.
+ */
+public class Packed64 extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ static final int BLOCK_SIZE = 64; // 32 = int, 64 = long
+ static final int BLOCK_BITS = 6; // The #bits representing BLOCK_SIZE
+ static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE
+
+ private static final int ENTRY_SIZE = BLOCK_SIZE + 1;
+ private static final int FAC_BITPOS = 3;
+
+ /*
+ * In order to make an efficient value-getter, conditionals should be
+ * avoided. A value can be positioned inside of a block, requiring shifting
+ * left or right or it can span two blocks, requiring a left-shift on the
+ * first block and a right-shift on the right block.
+ *
+ * By always shifting the first block both left and right, we get exactly
+ * the right bits. By always shifting the second block right and applying
+ * a mask, we get the right bits there. After that, we | the two bitsets.
+ */
+ private static final int[][] SHIFTS =
+ new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
+ //new int[BLOCK_SIZE+1][BLOCK_SIZE][BLOCK_SIZE+1];
+ private static final long[][] MASKS = new long[ENTRY_SIZE][ENTRY_SIZE];
+
+ static { // Generate shifts
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ int[] currentShifts = SHIFTS[elementBits];
+ int base = bitPos * FAC_BITPOS;
+ currentShifts[base ] = bitPos;
+ currentShifts[base + 1] = BLOCK_SIZE - elementBits;
+ if (bitPos <= BLOCK_SIZE - elementBits) { // Single block
+ currentShifts[base + 2] = 0;
+ MASKS[elementBits][bitPos] = 0;
+ } else { // Two blocks
+ int rBits = elementBits - (BLOCK_SIZE - bitPos);
+ currentShifts[base + 2] = BLOCK_SIZE - rBits;
+ MASKS[elementBits][bitPos] = ~(~0L << rBits);
+ }
+ }
+ }
+ }
+
+ /*
+ * The setter requires more masking than the getter.
+ */
+ private static final long[][] WRITE_MASKS =
+ new long[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
+ static {
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ long elementPosMask = ~(~0L << elementBits);
+ int[] currentShifts = SHIFTS[elementBits];
+ long[] currentMasks = WRITE_MASKS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ int base = bitPos * FAC_BITPOS;
+ currentMasks[base ] =~((elementPosMask
+ << currentShifts[base + 1])
+ >>> currentShifts[base]);
+ currentMasks[base+1] = ~(elementPosMask
+ << currentShifts[base + 2]);
+ currentMasks[base+2] = currentShifts[base + 2] == 0 ? 0 : ~0;
+ }
+ }
+ }
+
+ /* The bits */
+ private long[] blocks;
+
+ // Cached calculations
+ private int maxPos; // blocks.length * BLOCK_SIZE / elementBits - 1
+ private int[] shifts; // The shifts for the current elementBits
+ private long[] readMasks;
+ private long[] writeMasks;
+
+ /**
+ * Creates an array with the internal structures adjusted for the given
+ * limits and initialized to 0.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public Packed64(int valueCount, int bitsPerValue) {
+ // TODO: Test for edge-cases (2^31 values, 63 bitsPerValue)
+ // +2 due to the avoid-conditionals-trick. The last entry is always 0
+ this(new long[(int)((long)valueCount * bitsPerValue / BLOCK_SIZE + 2)],
+ valueCount, bitsPerValue);
+ }
+
+
+ /**
+ * Creates an array backed by the given blocks.
+ *
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the Packed32-structure.
+ * @param blocks used as the internal backing array. Not that the last
+ * element cannot be addressed directly.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public Packed64(long[] blocks, int valueCount, int bitsPerValue) {
+ super(valueCount, bitsPerValue);
+ this.blocks = blocks;
+ updateCached();
+ }
+
+ /**
+ * Creates an array with content retrieved from the given IndexInput.
+ * @param in an IndexInput, positioned at the start of Packed64-content.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ * @throws java.io.IOException if the values for the backing array could not
+ * be retrieved.
+ */
+ public Packed64(IndexInput in, int valueCount, int bitsPerValue)
+ throws IOException {
+ super(valueCount, bitsPerValue);
+ int size = size(bitsPerValue, valueCount);
+ blocks = new long[size+1]; // +1 due to non-conditional tricks
+ for(int i=0;i
+ * The implementation strives to avoid conditionals and expensive operations,
+ * sacrificing code clarity to achieve better performance.
+ *
+ * Space is optimally used within the boundaries of alignment, e.g.
+ * 7 bits/value fits 7 values/block for 64 bit.
+ * Bits are packed left-aligned to be bit pattern compatible with other bit
+ * array implementations where possible.
+ */
+public class PackedAligned64 extends PackedInts.ReaderImpl
+ implements PackedInts.Mutable {
+ static final int BLOCK_SIZE = 64; // 32 = int, 64 = long
+
+ private static final int ENTRY_SIZE = BLOCK_SIZE + 1;
+
+ /*
+ * A value is always positioned inside a single block, requiring a
+ * shift right to position the bits and a mask to extract them.
+ */
+ private static final int[][] SHIFTS = new int[ENTRY_SIZE][ENTRY_SIZE];
+ private static final long[] READ_MASKS = new long[ENTRY_SIZE];
+
+ static { // Generate shifts
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ int[] currentShifts = SHIFTS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ currentShifts[bitPos] = BLOCK_SIZE + bitPos - elementBits;
+ READ_MASKS[elementBits] = ~(~0L << elementBits);
+ }
+ }
+ }
+
+ /*
+ * Setting a value requires clearing the destination bits with a mask, then
+ * shifting the value to the left and or'ing the two numbers.
+ */
+ private static final long[][] WRITE_MASKS =
+ new long[ENTRY_SIZE][ENTRY_SIZE];
+ static {
+ for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
+ long elementPosMask = ~(~0L << elementBits);
+ int[] currentShifts = SHIFTS[elementBits];
+ long[] currentMasks = WRITE_MASKS[elementBits];
+ for (int bitPos = 0 ; bitPos < BLOCK_SIZE ; bitPos++) {
+ currentMasks[bitPos] = ~(elementPosMask
+ << currentShifts[bitPos]);
+ }
+ }
+ }
+
+ /* The bits */
+ private long[] blocks;
+
+ /* Cached values */
+ private int valuesPerBlock;
+ private int[] shifts;
+ private long readMask;
+ private long[] writeMasks;
+
+ /**
+ * Creates an array with the internal structures adjusted for the given
+ * limits and initialized to 0.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public PackedAligned64(int valueCount, int bitsPerValue) {
+ super(valueCount, bitsPerValue);
+ blocks = new long[size(valueCount, bitsPerValue)];
+ updateCached();
+ }
+
+ private static int size(int valueCount, int bitsPerValue) {
+ int valuesPerBlock = BLOCK_SIZE / bitsPerValue;
+ return valueCount / valuesPerBlock + 2;
+ }
+
+ /**
+ * Creates an array with content retrieved from the given IndexInput.
+ * @param in an IndexInput, positioned at the start of Packed64-content.
+ * @param valueCount the number of elements.
+ * @param bitsPerValue the number of bits available for any given value.
+ * @throws java.io.IOException if the values for the backing array could not
+ * be retrieved.
+ */
+ public PackedAligned64(IndexInput in, int valueCount, int bitsPerValue)
+ throws IOException {
+ super(valueCount, bitsPerValue);
+ int size = size(valueCount, bitsPerValue);
+ blocks = new long[size];
+ for(int i=0;i
+ * Note: The blocks are used directly, so changes to the given block will
+ * affect the Packed64-structure.
+ * @param blocks used as the internal backing array.
+ * @param valueCount the number of values.
+ * @param bitsPerValue the number of bits available for any given value.
+ */
+ public PackedAligned64(long[] blocks, int valueCount, int bitsPerValue) {
+ // TODO: Check that blocks.length is sufficient for holding length values
+ super(valueCount, bitsPerValue);
+ this.blocks = blocks;
+ updateCached();
+ }
+
+ private void updateCached() {
+ valuesPerBlock = BLOCK_SIZE / bitsPerValue;
+ shifts = SHIFTS[bitsPerValue];
+ readMask = READ_MASKS[bitsPerValue];
+ writeMasks = WRITE_MASKS[bitsPerValue];
+ }
+
+ /**
+ * @param index the position of the value.
+ * @return the value at the given index.
+ */
+ public long get(final int index) {
+ final int blockPos = index / valuesPerBlock;
+ final int bitPos = (index - (blockPos * valuesPerBlock)) * bitsPerValue;
+
+ return (blocks[blockPos] >>> shifts[bitPos]) & readMask;
+ }
+
+ public void set(final int index, final long value) {
+ final int blockPos = index / valuesPerBlock;
+ final int bitPos = (index - (blockPos * valuesPerBlock)) * bitsPerValue;
+
+ blocks[blockPos] = (blocks[blockPos] & writeMasks[bitPos])
+ | (value << shifts[bitPos]);
+ }
+
+ public void clear() {
+ Arrays.fill(blocks, 0);
+ }
+
+ public long ramBytesUsed() {
+ return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER
+ + blocks.length * RamUsageEstimator.NUM_BYTES_INT;
+ }
+
+}
\ No newline at end of file