Index: lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (revision 1222641) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/values/Writer.java (revision ) @@ -175,11 +175,12 @@ * the {@link Directory} to create the files from. * @param bytesUsed * a byte-usage tracking reference + * @param optimizeBytesDocValuesForSpeed Whether the space used for DV should be rounded up for higher lookup performance. * @return a new {@link Writer} instance for the given {@link Type} * @throws IOException */ public static Writer create(Type type, String id, Directory directory, - Comparator comp, Counter bytesUsed, IOContext context) throws IOException { + Comparator comp, Counter bytesUsed, IOContext context, boolean optimizeBytesDocValuesForSpeed) throws IOException { if (comp == null) { comp = BytesRef.getUTF8SortedAsUnicodeComparator(); } @@ -196,22 +197,22 @@ return Floats.getWriter(directory, id, bytesUsed, context, type); case BYTES_FIXED_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, true, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); case BYTES_FIXED_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, true, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); case BYTES_FIXED_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, true, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); case BYTES_VAR_STRAIGHT: return Bytes.getWriter(directory, id, Bytes.Mode.STRAIGHT, false, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); case BYTES_VAR_DEREF: return Bytes.getWriter(directory, id, Bytes.Mode.DEREF, false, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); case BYTES_VAR_SORTED: return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, false, comp, - bytesUsed, context); + bytesUsed, context, optimizeBytesDocValuesForSpeed); default: throw new IllegalArgumentException("Unknown Values: " + type); } Index: lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (revision 1222641) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/values/Bytes.java (revision ) @@ -114,13 +114,15 @@ * {@link Writer}. A call to {@link Writer#finish(int)} will release * all internally used resources and frees the memory tracking * reference. - * @param context + * @param sortedBytesFasterButMoreRam Whether the space used for DV should be rounded up for better lookup performance. + * @param context I/O Context * @return a new {@link Writer} instance * @throws IOException * if the files for the writer can not be created. */ public static Writer getWriter(Directory dir, String id, Mode mode, - boolean fixedSize, Comparator sortComparator, Counter bytesUsed, IOContext context) + boolean fixedSize, Comparator sortComparator, + Counter bytesUsed, IOContext context, boolean sortedBytesFasterButMoreRam) throws IOException { // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? @@ -134,7 +136,7 @@ } else if (mode == Mode.DEREF) { return new FixedDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context); + return new FixedSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, sortedBytesFasterButMoreRam); } } else { if (mode == Mode.STRAIGHT) { @@ -142,7 +144,7 @@ } else if (mode == Mode.DEREF) { return new VarDerefBytesImpl.Writer(dir, id, bytesUsed, context); } else if (mode == Mode.SORTED) { - return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context); + return new VarSortedBytesImpl.Writer(dir, id, sortComparator, bytesUsed, context, sortedBytesFasterButMoreRam); } } @@ -386,6 +388,7 @@ protected int lastDocId = -1; protected int[] docToEntry; protected final BytesRefHash hash; + protected boolean optimizePackedForSpeed = false; protected long maxBytes = 0; protected DerefBytesWriterBase(Directory dir, String id, String codecName, @@ -499,7 +502,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, int[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - PackedInts.bitsRequired(maxValue)); + bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -523,7 +526,7 @@ protected void writeIndex(IndexOutput idxOut, int docCount, long maxValue, long[] addresses, int[] toEntry) throws IOException { final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount, - PackedInts.bitsRequired(maxValue)); + bitsRequired(maxValue)); final int limit = docCount > docToEntry.length ? docToEntry.length : docCount; assert toEntry.length >= limit -1; @@ -542,6 +545,11 @@ w.add(0); } w.finish(); + } + + protected int bitsRequired(long maxValue){ + return optimizePackedForSpeed ? + PackedInts.getNextFixedSize(PackedInts.bitsRequired(maxValue)) : PackedInts.bitsRequired(maxValue); } } Index: lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (revision 1222641) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/values/FixedSortedBytesImpl.java (revision ) @@ -56,9 +56,10 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { + Counter bytesUsed, IOContext context, boolean optimizePackedForSpeed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); this.comp = comp; + this.optimizePackedForSpeed = optimizePackedForSpeed; } @Override Index: lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (revision 1222641) +++ lucene/src/java/org/apache/lucene/codecs/lucene40/values/VarSortedBytesImpl.java (revision ) @@ -57,10 +57,11 @@ private final Comparator comp; public Writer(Directory dir, String id, Comparator comp, - Counter bytesUsed, IOContext context) throws IOException { + Counter bytesUsed, IOContext context, boolean optimizePackedForSpeed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed, context); this.comp = comp; size = 0; + this.optimizePackedForSpeed = optimizePackedForSpeed; } @Override public void merge(MergeState mergeState, DocValues[] docValues) @@ -123,7 +124,7 @@ // total bytes of data idxOut.writeLong(maxBytes); PackedInts.Writer offsetWriter = PackedInts.getWriter(idxOut, count+1, - PackedInts.bitsRequired(maxBytes)); + bitsRequired(maxBytes)); // first dump bytes data, recording index & write offset as // we go final BytesRef spare = new BytesRef(); Index: lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java (revision 1222641) +++ lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java (revision ) @@ -63,7 +63,8 @@ Directory dir = newDirectory(); final Counter trackBytes = Counter.newCounter(); - Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random)); + Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random), + random.nextBoolean()); int maxDoc = 220; final String[] values = new String[maxDoc]; final int fixedLength = 1 + atLeast(50); Index: lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java (revision 1222641) +++ lucene/src/java/org/apache/lucene/codecs/DocValuesWriterBase.java (revision ) @@ -31,6 +31,7 @@ /** * Abstract base class for PerDocConsumer implementations + * * @lucene.experimental */ //TODO: this needs to go under lucene40 codec (its specific to its impl) @@ -39,12 +40,26 @@ protected final String segmentSuffix; private final Counter bytesUsed; protected final IOContext context; + private final boolean sortedBytesFasterButMoreRam; - + + /** + * @param state The state to initiate a {@link PerDocConsumer} instance + */ protected DocValuesWriterBase(PerDocWriteState state) { + this(state, true); + } + + /** + * @param state The state to initiate a {@link PerDocConsumer} instance + * @param sortedBytesFasterButMoreRam whether docvalues of type sorted bytes should be optimized for speed by rounding + * up the bytes used for a value to either 8, 16, 32 or 64 bytes. + */ + protected DocValuesWriterBase(PerDocWriteState state, boolean sortedBytesFasterButMoreRam) { this.segmentName = state.segmentName; this.segmentSuffix = state.segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; + this.sortedBytesFasterButMoreRam = sortedBytesFasterButMoreRam; } protected abstract Directory getDirectory() throws IOException; @@ -57,7 +72,7 @@ public DocValuesConsumer addValuesField(DocValues.Type valueType, FieldInfo field) throws IOException { return Writer.create(valueType, docValuesId(segmentName, field.number), - getDirectory(), getComparator(), bytesUsed, context); + getDirectory(), getComparator(), bytesUsed, context, sortedBytesFasterButMoreRam); } public static String docValuesId(String segmentsName, int fieldId) {