Index: lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (révision 1338745) +++ lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (copie de travail) @@ -198,6 +198,9 @@ packedInts.add(new Packed64(valueCount, bitsPerValue)); } packedInts.add(new Direct64(valueCount)); + if (Packed64SingleBlock.isSupported(bitsPerValue)) { + packedInts.add(Packed64SingleBlock.create(valueCount, bitsPerValue)); + } return packedInts; } @@ -286,5 +289,17 @@ p64.set(INDEX-1, 1); assertEquals("The value at position " + (INDEX-1) + " should be correct for Packed64", 1, p64.get(INDEX-1)); + p64 = null; + + for (int bits = 1; bits <=64; ++bits) { + if (Packed64SingleBlock.isSupported(bits)) { + int index = Integer.MAX_VALUE / bits + (bits == 1 ? 0 : 1); + Packed64SingleBlock p64sb = Packed64SingleBlock.create(index, bits); + p64sb.set(index - 1, 1); + assertEquals("The value at position " + (index-1) + + " should be correct for " + p64sb.getClass().getSimpleName(), + 1, p64sb.get(index-1)); + } + } } } Index: lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (révision 1338745) +++ lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (copie de travail) @@ -260,6 +260,7 @@ */ public static Mutable getMutable( int valueCount, int bitsPerValue) { + // TODO (?) return getMutable(valueCount, bitsPerValue, 0f); switch (bitsPerValue) { case 8: return new Direct8(valueCount); @@ -279,6 +280,52 @@ } /** + * Create a packed integer array with the given amount of values initialized + * to 0. the valueCount and the bitsPerValue cannot be changed after creation. + * All Mutables known by this factory are kept fully in RAM. + * + * @param valueCount the number of elements + * @param bitsPerValue the number of bits available for any given value + * @param acceptableOverheadPerValue an acceptable overhead per value in bits + * that can be used to select a faster implementation + * @return a mutable packed integer array + * @throws java.io.IOException if the Mutable could not be created. With the + * current implementations, this never happens, but the method + * signature allows for future persistence-backed Mutables. + * @lucene.internal + */ + public static Mutable getMutable(int valueCount, + int bitsPerValue, float acceptableOverheadPerValue) { + int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + if (bitsPerValue <= Byte.SIZE && maxBitsPerValue >= Byte.SIZE) { + return new Direct8(valueCount); + } else if (bitsPerValue <= Short.SIZE && maxBitsPerValue >= Short.SIZE) { + return new Direct16(valueCount); + } else if (bitsPerValue <= Integer.SIZE && maxBitsPerValue >= Integer.SIZE) { + return new Direct32(valueCount); + } else if (bitsPerValue <= Long.SIZE && maxBitsPerValue >= Long.SIZE) { + return new Direct64(valueCount); + } else { + if (bitsPerValue >= 32) { + return new Packed64(valueCount, bitsPerValue); + } else if (Constants.JRE_IS_64BIT) { + for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { + if (Packed64SingleBlock.isSupported(bpv)) { + float overhead = Packed64SingleBlock.overheadPerValue(bpv); + float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; + if (overhead <= acceptableOverhead) { + return Packed64SingleBlock.create(valueCount, bpv); + } + } + } + return new Packed64(valueCount, bitsPerValue); + } else { + return new Packed32(valueCount, bitsPerValue); + } + } + } + + /** * Create a packed integer array writer for the given number of values at the * given bits/value. Writers append to the given IndexOutput and has very * low memory overhead. Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) @@ -0,0 +1,362 @@ +package org.apache.lucene.util.packed; + +import java.util.Arrays; + +import org.apache.lucene.util.RamUsageEstimator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This class is similar to {@link Packed64} except that it trades space for + * speed by ensuring that a single block needs to be read in order to read/write + * a value. + */ +abstract class Packed64SingleBlock extends PackedInts.ReaderImpl implements + PackedInts.Mutable { + + private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 21}; + private static final long[][] WRITE_MASKS = new long[22][]; + private static final int[][] SHIFTS = new int[22][]; + static { + for (int bpv : SUPPORTED_BITS_PER_VALUE) { + initMasks(bpv); + } + } + + protected static void initMasks(int bpv) { + int valuesPerBlock = Long.SIZE / bpv; + long[] writeMasks = new long[valuesPerBlock]; + int[] shifts = new int[valuesPerBlock]; + long bits = (1L << bpv) - 1; + for (int i = 0; i < valuesPerBlock; ++i) { + shifts[i] = bpv * i; + writeMasks[i] = ~(bits << shifts[i]); + } + WRITE_MASKS[bpv] = writeMasks; + SHIFTS[bpv] = shifts; + } + + public static Packed64SingleBlock create(int valueCount, int bitsPerValue) { + switch (bitsPerValue) { + case 1: + return new Packed64SingleBlock1(valueCount); + case 2: + return new Packed64SingleBlock2(valueCount); + case 3: + return new Packed64SingleBlock3(valueCount); + case 4: + return new Packed64SingleBlock4(valueCount); + case 5: + return new Packed64SingleBlock5(valueCount); + case 6: + return new Packed64SingleBlock6(valueCount); + case 7: + return new Packed64SingleBlock7(valueCount); + case 9: + return new Packed64SingleBlock9(valueCount); + case 10: + return new Packed64SingleBlock10(valueCount); + case 12: + return new Packed64SingleBlock12(valueCount); + case 21: + return new Packed64SingleBlock21(valueCount); + default: + throw new IllegalArgumentException("Unsupported bitsPerValue: " + bitsPerValue); + } + } + + public static boolean isSupported(int bitsPerValue) { + return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + public static float overheadPerValue(int bitsPerValue) { + int valuesPerBlock = 64 / bitsPerValue; + int overhead = 64 % bitsPerValue; + return (float) overhead / valuesPerBlock; + } + + private final long[] blocks; + protected final int valuesPerBlock; + private final int[] shifts; + private final long[] writeMasks; + private final long readMask; + + Packed64SingleBlock(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + valuesPerBlock = Long.SIZE / bitsPerValue; + blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; + shifts = SHIFTS[bitsPerValue]; + writeMasks = WRITE_MASKS[bitsPerValue]; + readMask = ~writeMasks[0]; + } + + private static int requiredCapacity(int valueCount, int valuesPerBlock) { + return valueCount / valuesPerBlock + (valueCount % valuesPerBlock == 0 ? 0 : 1); + } + + protected int blockOffset(int offset) { + return offset / valuesPerBlock; + } + + protected int offsetInBlock(int offset) { + return offset % valuesPerBlock; + } + + @Override + public long get(int index) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + return (blocks[o] >> shifts[b]) & readMask; + } + + @Override + public void set(int index, long value) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + blocks[o] = (blocks[o] & writeMasks[b]) | (value << shifts[b]); + } + + @Override + public void clear() { + Arrays.fill(blocks, 0L); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return "PackedSingleBlock64(bitsPerValue=" + bitsPerValue + ", size=" + + size() + ", elements.length=" + blocks.length + ")"; + } +} + +// Specialisations that allow the JVM to optimize computation of the block +// offset as well as the offset in block + +final class Packed64SingleBlock21 extends Packed64SingleBlock { + + Packed64SingleBlock21(int valueCount) { + super(valueCount, 21); + assert valuesPerBlock == 3; + } + + @Override + protected int blockOffset(int offset) { + return offset / 3; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 3; + } + +} + +final class Packed64SingleBlock12 extends Packed64SingleBlock { + + Packed64SingleBlock12(int valueCount) { + super(valueCount, 12); + assert valuesPerBlock == 5; + } + + @Override + protected int blockOffset(int offset) { + return offset / 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 5; + } + +} + +final class Packed64SingleBlock10 extends Packed64SingleBlock { + + Packed64SingleBlock10(int valueCount) { + super(valueCount, 10); + assert valuesPerBlock == 6; + } + + @Override + protected int blockOffset(int offset) { + return offset / 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 6; + } + +} + +final class Packed64SingleBlock9 extends Packed64SingleBlock { + + Packed64SingleBlock9(int valueCount) { + super(valueCount, 9); + assert valuesPerBlock == 7; + } + + @Override + protected int blockOffset(int offset) { + return offset / 7; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 7; + } + +} + +final class Packed64SingleBlock7 extends Packed64SingleBlock { + + Packed64SingleBlock7(int valueCount) { + super(valueCount, 7); + assert valuesPerBlock == 9; + } + + @Override + protected int blockOffset(int offset) { + return offset / 9; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 9; + } + +} + +final class Packed64SingleBlock6 extends Packed64SingleBlock { + + Packed64SingleBlock6(int valueCount) { + super(valueCount, 6); + assert valuesPerBlock == 10; + } + + @Override + protected int blockOffset(int offset) { + return offset / 10; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 10; + } + +} + +final class Packed64SingleBlock5 extends Packed64SingleBlock { + + Packed64SingleBlock5(int valueCount) { + super(valueCount, 5); + assert valuesPerBlock == 12; + } + + @Override + protected int blockOffset(int offset) { + return offset / 12; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 12; + } + +} + +final class Packed64SingleBlock4 extends Packed64SingleBlock { + + Packed64SingleBlock4(int valueCount) { + super(valueCount, 4); + assert valuesPerBlock == 16; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 4; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 15; + } + +} + +final class Packed64SingleBlock3 extends Packed64SingleBlock { + + Packed64SingleBlock3(int valueCount) { + super(valueCount, 3); + assert valuesPerBlock == 21; + } + + @Override + protected int blockOffset(int offset) { + return offset / 21; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 21; + } + +} + +final class Packed64SingleBlock2 extends Packed64SingleBlock { + + Packed64SingleBlock2(int valueCount) { + super(valueCount, 2); + assert valuesPerBlock == 32; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 31; + } + +} + +final class Packed64SingleBlock1 extends Packed64SingleBlock { + + Packed64SingleBlock1(int valueCount) { + super(valueCount, 1); + assert valuesPerBlock == 64; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 63; + } + +} \ No newline at end of file