Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (révision 1339164) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/JoinDocFreqValueSource.java (copie de travail) @@ -27,6 +27,7 @@ import org.apache.lucene.search.FieldCache.DocTerms; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ReaderUtil; +import org.apache.lucene.util.packed.PackedInts; /** * Use a field value and find the Document Frequency within another field. @@ -52,7 +53,7 @@ @Override public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { - final DocTerms terms = cache.getTerms(readerContext.reader(), field, true ); + final DocTerms terms = cache.getTerms(readerContext.reader(), field, PackedInts.FAST ); final IndexReader top = ReaderUtil.getTopLevelContext(readerContext).reader(); return new IntDocValues(this) { Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (révision 1339164) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (copie de travail) @@ -64,6 +64,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.packed.PackedInts; public class TestIndexWriter extends LuceneTestCase { @@ -1677,7 +1678,7 @@ w.close(); assertEquals(1, reader.docFreq(new Term("content", bigTerm))); - FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(reader), "content", random().nextBoolean()); + FieldCache.DocTermsIndex dti = FieldCache.DEFAULT.getTermsIndex(SlowCompositeReaderWrapper.wrap(reader), "content", random().nextFloat() * PackedInts.FAST); assertEquals(5, dti.numOrd()); // +1 for null ord assertEquals(4, dti.size()); assertEquals(bigTermBytesRef, dti.lookup(3, new BytesRef())); Index: lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (révision 1339164) +++ lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (copie de travail) @@ -198,6 +198,9 @@ packedInts.add(new Packed64(valueCount, bitsPerValue)); } packedInts.add(new Direct64(valueCount)); + if (Packed64SingleBlock.isSupported(bitsPerValue)) { + packedInts.add(Packed64SingleBlock.create(valueCount, bitsPerValue)); + } return packedInts; } @@ -286,5 +289,17 @@ p64.set(INDEX-1, 1); assertEquals("The value at position " + (INDEX-1) + " should be correct for Packed64", 1, p64.get(INDEX-1)); + p64 = null; + + for (int bits = 1; bits <=64; ++bits) { + if (Packed64SingleBlock.isSupported(bits)) { + int index = Integer.MAX_VALUE / bits + (bits == 1 ? 0 : 1); + Packed64SingleBlock p64sb = Packed64SingleBlock.create(index, bits); + p64sb.set(index - 1, 1); + assertEquals("The value at position " + (index-1) + + " should be correct for " + p64sb.getClass().getSimpleName(), + 1, p64sb.get(index-1)); + } + } } } Index: lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java (révision 1339164) +++ lucene/core/src/test/org/apache/lucene/codecs/lucene40/values/TestDocValues.java (copie de travail) @@ -40,6 +40,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.packed.PackedInts; // TODO: some of this should be under lucene40 codec tests? is talking to codec directly?f public class TestDocValues extends LuceneTestCase { @@ -71,7 +72,7 @@ Directory dir = newDirectory(); final Counter trackBytes = Counter.newCounter(); DocValuesConsumer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random()), - random().nextBoolean()); + random().nextFloat() * PackedInts.FAST); int maxDoc = 220; final String[] values = new String[maxDoc]; final int fixedLength = 1 + atLeast(50); Index: lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (copie de travail) @@ -1071,14 +1071,12 @@ } } - private static boolean DEFAULT_FASTER_BUT_MORE_RAM = true; - public DocTermsIndex getTermsIndex(AtomicReader reader, String field) throws IOException { - return getTermsIndex(reader, field, DEFAULT_FASTER_BUT_MORE_RAM); + return getTermsIndex(reader, field, PackedInts.DEFAULT); } - public DocTermsIndex getTermsIndex(AtomicReader reader, String field, boolean fasterButMoreRAM) throws IOException { - return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)), false); + public DocTermsIndex getTermsIndex(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException { + return (DocTermsIndex) caches.get(DocTermsIndex.class).get(reader, new Entry(field, acceptableOverheadRatio), false); } static class DocTermsIndexCache extends Cache { @@ -1092,7 +1090,7 @@ Terms terms = reader.terms(entryKey.field); - final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue(); + final float acceptableOverheadRatio = ((Float) entryKey.custom).floatValue(); final PagedBytes bytes = new PagedBytes(15); @@ -1142,8 +1140,8 @@ startNumUniqueTerms = 1; } - GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1+startNumUniqueTerms, fasterButMoreRAM); - final GrowableWriter docToTermOrd = new GrowableWriter(startTermsBPV, maxDoc, fasterButMoreRAM); + GrowableWriter termOrdToBytesOffset = new GrowableWriter(startBytesBPV, 1+startNumUniqueTerms, acceptableOverheadRatio); + final GrowableWriter docToTermOrd = new GrowableWriter(startTermsBPV, maxDoc, acceptableOverheadRatio); // 0 is reserved for "unset" bytes.copyUsingLengthPrefix(new BytesRef()); @@ -1219,11 +1217,11 @@ // TODO: this if DocTermsIndex was already created, we // should share it... public DocTerms getTerms(AtomicReader reader, String field) throws IOException { - return getTerms(reader, field, DEFAULT_FASTER_BUT_MORE_RAM); + return getTerms(reader, field, PackedInts.DEFAULT); } - public DocTerms getTerms(AtomicReader reader, String field, boolean fasterButMoreRAM) throws IOException { - return (DocTerms) caches.get(DocTerms.class).get(reader, new Entry(field, Boolean.valueOf(fasterButMoreRAM)), false); + public DocTerms getTerms(AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException { + return (DocTerms) caches.get(DocTerms.class).get(reader, new Entry(field, acceptableOverheadRatio), false); } static final class DocTermsCache extends Cache { @@ -1237,7 +1235,7 @@ Terms terms = reader.terms(entryKey.field); - final boolean fasterButMoreRAM = ((Boolean) entryKey.custom).booleanValue(); + final float acceptableOverheadRatio = ((Float) entryKey.custom).floatValue(); final int termCountHardLimit = reader.maxDoc(); @@ -1268,7 +1266,7 @@ startBPV = 1; } - final GrowableWriter docToOffset = new GrowableWriter(startBPV, reader.maxDoc(), fasterButMoreRAM); + final GrowableWriter docToOffset = new GrowableWriter(startBPV, reader.maxDoc(), acceptableOverheadRatio); // pointer==0 means not set bytes.copyUsingLengthPrefix(new BytesRef()); Index: lucene/core/src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCache.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/search/FieldCache.java (copie de travail) @@ -494,7 +494,7 @@ * faster lookups (default is "true"). Note that the * first call for a given reader and field "wins", * subsequent calls will share the same cache entry. */ - public DocTerms getTerms (AtomicReader reader, String field, boolean fasterButMoreRAM) + public DocTerms getTerms (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException; /** Returned by {@link #getTermsIndex} */ @@ -571,7 +571,7 @@ * faster lookups (default is "true"). Note that the * first call for a given reader and field "wins", * subsequent calls will share the same cache entry. */ - public DocTermsIndex getTermsIndex (AtomicReader reader, String field, boolean fasterButMoreRAM) + public DocTermsIndex getTermsIndex (AtomicReader reader, String field, float acceptableOverheadRatio) throws IOException; /** Index: lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (copie de travail) @@ -28,22 +28,14 @@ private long currentMaxValue; private PackedInts.Mutable current; - private final boolean roundFixedSize; + private final float acceptableOverheadRatio; - public GrowableWriter(int startBitsPerValue, int valueCount, boolean roundFixedSize) { - this.roundFixedSize = roundFixedSize; - current = PackedInts.getMutable(valueCount, getSize(startBitsPerValue)); + public GrowableWriter(int startBitsPerValue, int valueCount, float acceptableOverheadRatio) { + this.acceptableOverheadRatio = acceptableOverheadRatio; + current = PackedInts.getMutable(valueCount, startBitsPerValue, this.acceptableOverheadRatio); currentMaxValue = PackedInts.maxValue(current.getBitsPerValue()); } - private final int getSize(int bpv) { - if (roundFixedSize) { - return PackedInts.getNextFixedSize(bpv); - } else { - return bpv; - } - } - public long get(int index) { return current.get(index); } @@ -78,7 +70,7 @@ currentMaxValue *= 2; } final int valueCount = size(); - PackedInts.Mutable next = PackedInts.getMutable(valueCount, getSize(bpv)); + PackedInts.Mutable next = PackedInts.getMutable(valueCount, bpv, acceptableOverheadRatio); for(int i=0;iacceptableOverheadRatio will trade space + * for speed by selecting a faster but potentially less memory-efficient + * implementation. An acceptableOverheadRatio of + * {@link PackedInts#COMPACT} will make sure that the most memory-efficient + * implementation is selected whereas {@link PackedInts#FAST} will make sure + * that the fastest implementation is selected. + * + * @param valueCount the number of elements + * @param bitsPerValue the number of bits available for any given value + * @param acceptableOverheadRatio an acceptable overhead ratio per value + * @return a mutable packed integer array * @throws java.io.IOException if the Mutable could not be created. With the * current implementations, this never happens, but the method * signature allows for future persistence-backed Mutables. * @lucene.internal */ - public static Mutable getMutable( - int valueCount, int bitsPerValue) { - switch (bitsPerValue) { - case 8: + public static Mutable getMutable(int valueCount, + int bitsPerValue, float acceptableOverheadRatio) { + acceptableOverheadRatio = Math.max(COMPACT, acceptableOverheadRatio); + acceptableOverheadRatio = Math.min(FAST, acceptableOverheadRatio); + float acceptableOverheadPerValue = acceptableOverheadRatio * bitsPerValue; // in bits + + int maxBitsPerValue = bitsPerValue + (int) acceptableOverheadPerValue; + + if (bitsPerValue <= Byte.SIZE && maxBitsPerValue >= Byte.SIZE) { return new Direct8(valueCount); - case 16: + } else if (bitsPerValue <= Short.SIZE && maxBitsPerValue >= Short.SIZE) { return new Direct16(valueCount); - case 32: + } else if (bitsPerValue <= Integer.SIZE && maxBitsPerValue >= Integer.SIZE) { return new Direct32(valueCount); - case 64: + } else if (bitsPerValue <= Long.SIZE && maxBitsPerValue >= Long.SIZE) { return new Direct64(valueCount); - default: - if (Constants.JRE_IS_64BIT || bitsPerValue >= 32) { + } else { + if (bitsPerValue >= 32) { return new Packed64(valueCount, bitsPerValue); + } else if (Constants.JRE_IS_64BIT) { + for (int bpv = bitsPerValue; bpv <= maxBitsPerValue; ++bpv) { + if (Packed64SingleBlock.isSupported(bpv)) { + float overhead = Packed64SingleBlock.overheadPerValue(bpv); + float acceptableOverhead = acceptableOverheadPerValue + bitsPerValue - bpv; + if (overhead <= acceptableOverhead) { + return Packed64SingleBlock.create(valueCount, bpv); + } + } + } + return new Packed64(valueCount, bitsPerValue); } else { return new Packed32(valueCount, bitsPerValue); } @@ -322,25 +362,4 @@ return bitsPerValue == 64 ? Long.MAX_VALUE : ~(~0L << bitsPerValue); } - /** Rounds bitsPerValue up to 8, 16, 32 or 64. */ - public static int getNextFixedSize(int bitsPerValue) { - if (bitsPerValue <= 8) { - return 8; - } else if (bitsPerValue <= 16) { - return 16; - } else if (bitsPerValue <= 32) { - return 32; - } else { - return 64; - } - } - - /** Possibly wastes some storage in exchange for faster lookups */ - public static int getRoundedFixedSize(int bitsPerValue) { - if (bitsPerValue > 58 || (bitsPerValue < 32 && bitsPerValue > 29)) { // 10% space-waste is ok - return getNextFixedSize(bitsPerValue); - } else { - return bitsPerValue; - } - } } Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (révision 0) @@ -0,0 +1,362 @@ +package org.apache.lucene.util.packed; + +import java.util.Arrays; + +import org.apache.lucene.util.RamUsageEstimator; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to You under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** + * This class is similar to {@link Packed64} except that it trades space for + * speed by ensuring that a single block needs to be read in order to read/write + * a value. + */ +abstract class Packed64SingleBlock extends PackedInts.ReaderImpl implements + PackedInts.Mutable { + + private static final int[] SUPPORTED_BITS_PER_VALUE = new int[] {1, 2, 3, 4, 5, 6, 7, 9, 10, 12, 21}; + private static final long[][] WRITE_MASKS = new long[22][]; + private static final int[][] SHIFTS = new int[22][]; + static { + for (int bpv : SUPPORTED_BITS_PER_VALUE) { + initMasks(bpv); + } + } + + protected static void initMasks(int bpv) { + int valuesPerBlock = Long.SIZE / bpv; + long[] writeMasks = new long[valuesPerBlock]; + int[] shifts = new int[valuesPerBlock]; + long bits = (1L << bpv) - 1; + for (int i = 0; i < valuesPerBlock; ++i) { + shifts[i] = bpv * i; + writeMasks[i] = ~(bits << shifts[i]); + } + WRITE_MASKS[bpv] = writeMasks; + SHIFTS[bpv] = shifts; + } + + public static Packed64SingleBlock create(int valueCount, int bitsPerValue) { + switch (bitsPerValue) { + case 1: + return new Packed64SingleBlock1(valueCount); + case 2: + return new Packed64SingleBlock2(valueCount); + case 3: + return new Packed64SingleBlock3(valueCount); + case 4: + return new Packed64SingleBlock4(valueCount); + case 5: + return new Packed64SingleBlock5(valueCount); + case 6: + return new Packed64SingleBlock6(valueCount); + case 7: + return new Packed64SingleBlock7(valueCount); + case 9: + return new Packed64SingleBlock9(valueCount); + case 10: + return new Packed64SingleBlock10(valueCount); + case 12: + return new Packed64SingleBlock12(valueCount); + case 21: + return new Packed64SingleBlock21(valueCount); + default: + throw new IllegalArgumentException("Unsupported bitsPerValue: " + bitsPerValue); + } + } + + public static boolean isSupported(int bitsPerValue) { + return Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsPerValue) >= 0; + } + + public static float overheadPerValue(int bitsPerValue) { + int valuesPerBlock = 64 / bitsPerValue; + int overhead = 64 % bitsPerValue; + return (float) overhead / valuesPerBlock; + } + + private final long[] blocks; + protected final int valuesPerBlock; + private final int[] shifts; + private final long[] writeMasks; + private final long readMask; + + Packed64SingleBlock(int valueCount, int bitsPerValue) { + super(valueCount, bitsPerValue); + valuesPerBlock = Long.SIZE / bitsPerValue; + blocks = new long[requiredCapacity(valueCount, valuesPerBlock)]; + shifts = SHIFTS[bitsPerValue]; + writeMasks = WRITE_MASKS[bitsPerValue]; + readMask = ~writeMasks[0]; + } + + private static int requiredCapacity(int valueCount, int valuesPerBlock) { + return valueCount / valuesPerBlock + (valueCount % valuesPerBlock == 0 ? 0 : 1); + } + + protected int blockOffset(int offset) { + return offset / valuesPerBlock; + } + + protected int offsetInBlock(int offset) { + return offset % valuesPerBlock; + } + + @Override + public long get(int index) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + return (blocks[o] >> shifts[b]) & readMask; + } + + @Override + public void set(int index, long value) { + final int o = blockOffset(index); + final int b = offsetInBlock(index); + + blocks[o] = (blocks[o] & writeMasks[b]) | (value << shifts[b]); + } + + @Override + public void clear() { + Arrays.fill(blocks, 0L); + } + + public long ramBytesUsed() { + return RamUsageEstimator.sizeOf(blocks); + } + + @Override + public String toString() { + return "PackedSingleBlock64(bitsPerValue=" + bitsPerValue + ", size=" + + size() + ", elements.length=" + blocks.length + ")"; + } + + //Specialisations that allow the JVM to optimize computation of the block + //offset as well as the offset in block + + static final class Packed64SingleBlock21 extends Packed64SingleBlock { + + Packed64SingleBlock21(int valueCount) { + super(valueCount, 21); + assert valuesPerBlock == 3; + } + + @Override + protected int blockOffset(int offset) { + return offset / 3; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 3; + } + + } + + static final class Packed64SingleBlock12 extends Packed64SingleBlock { + + Packed64SingleBlock12(int valueCount) { + super(valueCount, 12); + assert valuesPerBlock == 5; + } + + @Override + protected int blockOffset(int offset) { + return offset / 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 5; + } + + } + + static final class Packed64SingleBlock10 extends Packed64SingleBlock { + + Packed64SingleBlock10(int valueCount) { + super(valueCount, 10); + assert valuesPerBlock == 6; + } + + @Override + protected int blockOffset(int offset) { + return offset / 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 6; + } + + } + + static final class Packed64SingleBlock9 extends Packed64SingleBlock { + + Packed64SingleBlock9(int valueCount) { + super(valueCount, 9); + assert valuesPerBlock == 7; + } + + @Override + protected int blockOffset(int offset) { + return offset / 7; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 7; + } + + } + + static final class Packed64SingleBlock7 extends Packed64SingleBlock { + + Packed64SingleBlock7(int valueCount) { + super(valueCount, 7); + assert valuesPerBlock == 9; + } + + @Override + protected int blockOffset(int offset) { + return offset / 9; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 9; + } + + } + + static final class Packed64SingleBlock6 extends Packed64SingleBlock { + + Packed64SingleBlock6(int valueCount) { + super(valueCount, 6); + assert valuesPerBlock == 10; + } + + @Override + protected int blockOffset(int offset) { + return offset / 10; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 10; + } + + } + + static final class Packed64SingleBlock5 extends Packed64SingleBlock { + + Packed64SingleBlock5(int valueCount) { + super(valueCount, 5); + assert valuesPerBlock == 12; + } + + @Override + protected int blockOffset(int offset) { + return offset / 12; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 12; + } + + } + + static final class Packed64SingleBlock4 extends Packed64SingleBlock { + + Packed64SingleBlock4(int valueCount) { + super(valueCount, 4); + assert valuesPerBlock == 16; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 4; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 15; + } + + } + + static final class Packed64SingleBlock3 extends Packed64SingleBlock { + + Packed64SingleBlock3(int valueCount) { + super(valueCount, 3); + assert valuesPerBlock == 21; + } + + @Override + protected int blockOffset(int offset) { + return offset / 21; + } + + @Override + protected int offsetInBlock(int offset) { + return offset % 21; + } + + } + + static final class Packed64SingleBlock2 extends Packed64SingleBlock { + + Packed64SingleBlock2(int valueCount) { + super(valueCount, 2); + assert valuesPerBlock == 32; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 5; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 31; + } + + } + + static final class Packed64SingleBlock1 extends Packed64SingleBlock { + + Packed64SingleBlock1(int valueCount) { + super(valueCount, 1); + assert valuesPerBlock == 64; + } + + @Override + protected int blockOffset(int offset) { + return offset >> 6; + } + + @Override + protected int offsetInBlock(int offset) { + return offset & 63; + } + + } +} \ No newline at end of file Index: lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene3x/TermInfosReaderIndex.java (copie de travail) @@ -74,7 +74,7 @@ PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput(); final int bitEstimate = 1+MathUtil.log(tiiFileLength, 2); - GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, false); + GrowableWriter indexToTerms = new GrowableWriter(bitEstimate, indexSize, PackedInts.DEFAULT); String currentField = null; List fieldStrs = new ArrayList(); Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (copie de travail) @@ -125,7 +125,7 @@ */ public static DocValuesConsumer getWriter(Directory dir, String id, Mode mode, boolean fixedSize, Comparator sortComparator, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? @@ -139,7 +139,7 @@ } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); + return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, acceptableOverheadRatio); } } else { if (mode == Mode.STRAIGHT) { @@ -147,7 +147,7 @@ } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, fasterButMoreRam); + return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, acceptableOverheadRatio); } } @@ -374,32 +374,32 @@ protected int lastDocId = -1; protected int[] docToEntry; protected final BytesRefHash hash; - protected final boolean fasterButMoreRam; + protected final float acceptableOverheadRatio; protected long maxBytes = 0; protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Counter bytesUsed, IOContext context, Type type) throws IOException { this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( - ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, false, type); + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, PackedInts.DEFAULT, type); } protected DerefBytesWriterBase(Directory dir, String id, String codecName, - int codecVersion, Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type) + int codecVersion, Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) throws IOException { this(dir, id, codecName, codecVersion, new DirectTrackingAllocator( - ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, fasterButMoreRam,type); + ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed, context, acceptableOverheadRatio,type); } protected DerefBytesWriterBase(Directory dir, String id, String codecName, int codecVersion, Allocator allocator, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam, Type type) throws IOException { + Counter bytesUsed, IOContext context, float acceptableOverheadRatio, Type type) throws IOException { super(dir, id, codecName, codecVersion, bytesUsed, context, type); hash = new BytesRefHash(new ByteBlockPool(allocator), BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray( BytesRefHash.DEFAULT_CAPACITY, bytesUsed)); docToEntry = new int[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); - this.fasterButMoreRam = fasterButMoreRam; + this.acceptableOverheadRatio = acceptableOverheadRatio; } protected static int writePrefixLength(DataOutput datOut, BytesRef bytes) @@ -498,7 +498,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, int[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - bitsRequired(maxValue)); + PackedInts.bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -522,7 +522,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, long[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - bitsRequired(maxValue)); + PackedInts.bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -542,11 +542,6 @@ } w.finish(); } - - protected int bitsRequired(long maxValue){ - return fasterButMoreRam ? - PackedInts.getNextFixedSize(PackedInts.bitsRequired(maxValue)) : PackedInts.bitsRequired(maxValue); - } } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (copie de travail) @@ -57,8 +57,8 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_FIXED_SORTED); + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_FIXED_SORTED); this.comp = comp; } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (copie de travail) @@ -84,7 +84,7 @@ * @throws IOException */ public static DocValuesConsumer create(Type type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { if (comp == null) { comp = BytesRef.getUTF8SortedAsUnicodeComparator(); } @@ -101,22 +101,22 @@ return Floats.getWriter(directory, id, bytesUsed, context, type); case BYTES_FIXED_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_FIXED_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_FIXED_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); case BYTES_VAR_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp, - bytesUsed, context, fasterButMoreRam); + bytesUsed, context, acceptableOverheadRatio); default: throw new IllegalArgumentException("Unknown Values: " + type); } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/DocValuesWriterBase.java (copie de travail) @@ -31,6 +31,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; +import org.apache.lucene.util.packed.PackedInts; /** * Abstract base class for PerDocConsumer implementations @@ -41,7 +42,7 @@ protected final String segmentName; private final Counter bytesUsed; protected final IOContext context; - private final boolean fasterButMoreRam; + private final float acceptableOverheadRatio; /** * Filename extension for index files @@ -57,7 +58,7 @@ * @param state The state to initiate a {@link PerDocConsumer} instance */ protected DocValuesWriterBase(PerDocWriteState state) { - this(state, true); + this(state, PackedInts.FAST); } /** @@ -66,11 +67,11 @@ * used for a value to either 8, 16, 32 or 64 bytes. This option is only applicable for * docvalues of type {@link Type#BYTES_FIXED_SORTED} and {@link Type#BYTES_VAR_SORTED}. */ - protected DocValuesWriterBase(PerDocWriteState state, boolean fasterButMoreRam) { + protected DocValuesWriterBase(PerDocWriteState state, float acceptableOverheadRatio) { this.segmentName = state.segmentName; this.bytesUsed = state.bytesUsed; this.context = state.context; - this.fasterButMoreRam = fasterButMoreRam; + this.acceptableOverheadRatio = acceptableOverheadRatio; } protected abstract Directory getDirectory() throws IOException; @@ -83,7 +84,7 @@ public DocValuesConsumer addValuesField(Type valueType, FieldInfo field) throws IOException { return Writer.create(valueType, PerDocProducerBase.docValuesId(segmentName, field.number), - getDirectory(), getComparator(), bytesUsed, context, fasterButMoreRam); + getDirectory(), getComparator(), bytesUsed, context, acceptableOverheadRatio); } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (révision 1339164) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (copie de travail) @@ -58,8 +58,8 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context, boolean fasterButMoreRam) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, fasterButMoreRam, Type.BYTES_VAR_SORTED); + Counter bytesUsed, IOContext context, float acceptableOverheadRatio) throws IOException { + super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context, acceptableOverheadRatio, Type.BYTES_VAR_SORTED); this.comp = comp; size = 0; } @@ -125,7 +125,7 @@ // total bytes of data idxOut.writeLong(maxBytes); PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1, - bitsRequired(maxBytes)); + PackedInts.bitsRequired(maxBytes)); // first dump bytes data, recording index & write offset as // we go final BytesRef spare = new BytesRef();