Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (révision 1469808) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (copie de travail) @@ -17,6 +17,11 @@ * limitations under the License. */ +import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesConsumer.TABLE_COMPRESSED; +import static org.apache.lucene.codecs.lucene42.Lucene42DocValuesConsumer.UNCOMPRESSED; + import java.io.IOException; import java.util.Comparator; import java.util.HashMap; @@ -80,10 +85,11 @@ // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; + final int version; try { - CodecUtil.checkHeader(in, metaCodec, + version = CodecUtil.checkHeader(in, metaCodec, Lucene42DocValuesConsumer.VERSION_START, - Lucene42DocValuesConsumer.VERSION_START); + Lucene42DocValuesConsumer.VERSION_CURRENT); numerics = new HashMap(); binaries = new HashMap(); fsts = new HashMap(); @@ -99,9 +105,12 @@ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); - CodecUtil.checkHeader(data, dataCodec, - Lucene42DocValuesConsumer.VERSION_START, - Lucene42DocValuesConsumer.VERSION_START); + final int version2 = CodecUtil.checkHeader(data, dataCodec, + Lucene42DocValuesConsumer.VERSION_START, + Lucene42DocValuesConsumer.VERSION_CURRENT); + if (version != version2) { + throw new CorruptIndexException("Format versions mismatch"); + } } private void readFields(IndexInput meta, FieldInfos infos) throws IOException { @@ -152,41 +161,53 @@ private NumericDocValues loadNumeric(FieldInfo field) throws IOException { NumericEntry entry = numerics.get(field.number); data.seek(entry.offset); - if (entry.format == Lucene42DocValuesConsumer.TABLE_COMPRESSED) { - int size = data.readVInt(); - final long decode[] = new long[size]; - for (int i = 0; i < decode.length; i++) { - decode[i] = data.readLong(); - } - final int formatID = data.readVInt(); - final int bitsPerValue = data.readVInt(); - final PackedInts.Reader reader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); - return new NumericDocValues() { - @Override - public long get(int docID) { - return decode[(int)reader.get(docID)]; + switch (entry.format) { + case TABLE_COMPRESSED: + int size = data.readVInt(); + final long decode[] = new long[size]; + for (int i = 0; i < decode.length; i++) { + decode[i] = data.readLong(); } - }; - } else if (entry.format == Lucene42DocValuesConsumer.DELTA_COMPRESSED) { - final int blockSize = data.readVInt(); - final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); - return new NumericDocValues() { - @Override - public long get(int docID) { - return reader.get(docID); - } - }; - } else if (entry.format == Lucene42DocValuesConsumer.UNCOMPRESSED) { - final byte bytes[] = new byte[maxDoc]; - data.readBytes(bytes, 0, bytes.length); - return new NumericDocValues() { - @Override - public long get(int docID) { - return bytes[docID]; - } - }; - } else { - throw new IllegalStateException(); + final int formatID = data.readVInt(); + final int bitsPerValue = data.readVInt(); + final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); + return new NumericDocValues() { + @Override + public long get(int docID) { + return decode[(int)ordsReader.get(docID)]; + } + }; + case DELTA_COMPRESSED: + final int blockSize = data.readVInt(); + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); + return new NumericDocValues() { + @Override + public long get(int docID) { + return reader.get(docID); + } + }; + case UNCOMPRESSED: + final byte bytes[] = new byte[maxDoc]; + data.readBytes(bytes, 0, bytes.length); + return new NumericDocValues() { + @Override + public long get(int docID) { + return bytes[docID]; + } + }; + case GCD_COMPRESSED: + final long min = data.readLong(); + final long mult = data.readLong(); + final int quotientBlockSize = data.readVInt(); + final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); + return new NumericDocValues() { + @Override + public long get(int docID) { + return min + mult * quotientReader.get(docID); + } + }; + default: + throw new CorruptIndexException("Unknown format: " + entry.format); } } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java (révision 1469808) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java (copie de travail) @@ -44,6 +44,8 @@ *
  • Uncompressed Numerics: when all values would fit into a single byte, and the * acceptableOverheadRatio would pack values into 8 bits per value anyway, they * are written as absolute values (with no indirection or packing) for performance. + *
  • GCD-compressed Numerics: when all numbers share a common divisor, such as dates, the greatest + * common denominator (GCD) is computed, and quotients are stored using Delta-compressed Numerics. *
  • Fixed-width Binary: one large concatenated byte[] is written, along with the fixed length. * Each document's value can be addressed by maxDoc*length. *
  • Variable-width Binary: one large concatenated byte[] is written, along with end addresses @@ -93,6 +95,8 @@ *
  • 2 --> uncompressed. When the acceptableOverheadRatio parameter would upgrade the number * of bits required to 8, and all values fit in a byte, these are written as absolute binary values * for performance. + *
  • 3 -->, gcd-compressed. When all integers share a common divisor, only quotients are stored + * using blocks of delta-encoded ints. * *

    MinLength and MaxLength represent the min and max byte[] value lengths for Binary values. * If they are equal, then all values are of a fixed size, and can be addressed as DataOffset + (docID * length). @@ -103,7 +107,7 @@ *

    For DocValues field, this stores the actual per-document data (the heavy-lifting)

    *

    DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData>NumFields

    *
      - *
    • NumericData --> DeltaCompressedNumerics | TableCompressedNumerics | UncompressedNumerics
      • + *
    • NumericData --> DeltaCompressedNumerics | TableCompressedNumerics | UncompressedNumerics | GCDCompressedNumerics
      • *
    • BinaryData --> {@link DataOutput#writeByte Byte}DataLength,Addresses
      • *
    • SortedData --> {@link FST FST<Int64>}
      • *
    • DeltaCompressedNumerics --> {@link BlockPackedWriter BlockPackedInts(blockSize=4096)}
      • Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (révision 1469808) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (copie de travail) @@ -34,6 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.MathUtil; import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.FST.INPUT_TYPE; @@ -49,7 +50,8 @@ */ class Lucene42DocValuesConsumer extends DocValuesConsumer { static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; + static final int VERSION_GCD_COMPRESSION = 1; + static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION; static final byte NUMBER = 0; static final byte BYTES = 1; @@ -60,6 +62,7 @@ static final byte DELTA_COMPRESSED = 0; static final byte TABLE_COMPRESSED = 1; static final byte UNCOMPRESSED = 2; + static final byte GCD_COMPRESSED = 3; final IndexOutput data, meta; final int maxDoc; @@ -83,27 +86,53 @@ } } } - + @Override public void addNumericField(FieldInfo field, Iterable values) throws IOException { + addNumericField(field, values, true); + } + + void addNumericField(FieldInfo field, Iterable values, boolean optimizeStorage) throws IOException { meta.writeVInt(field.number); meta.writeByte(NUMBER); meta.writeLong(data.getFilePointer()); long minValue = Long.MAX_VALUE; long maxValue = Long.MIN_VALUE; + long gcd = 0; // TODO: more efficient? - HashSet uniqueValues = new HashSet(); - for(Number nv : values) { - long v = nv.longValue(); - minValue = Math.min(minValue, v); - maxValue = Math.max(maxValue, v); - if (uniqueValues != null) { - if (uniqueValues.add(v)) { - if (uniqueValues.size() > 256) { - uniqueValues = null; + HashSet uniqueValues = null; + if (optimizeStorage) { + uniqueValues = new HashSet<>(); + + long count = 0; + for (Number nv : values) { + final long v = nv.longValue(); + + if (gcd != 1) { + if (v < - Long.MAX_VALUE / 2 || v > Long.MAX_VALUE / 2) { + // in that case v - minValue might overflow and make the GCD computation return + // wrong results. Since these extreme values are unlikely, we just discard + // GCD computation for them + gcd = 1; + } else if (count != 0) { // minValue needs to be set first + gcd = MathUtil.gcd(gcd, v - minValue); } } + + minValue = Math.min(minValue, v); + maxValue = Math.max(maxValue, v); + + if (uniqueValues != null) { + if (uniqueValues.add(v)) { + if (uniqueValues.size() > 256) { + uniqueValues = null; + } + } + } + + ++count; } + assert count == maxDoc; } if (uniqueValues != null) { @@ -135,6 +164,18 @@ } writer.finish(); } + } else if (gcd != 0 && gcd != 1) { + meta.writeByte(GCD_COMPRESSED); + meta.writeVInt(PackedInts.VERSION_CURRENT); + data.writeLong(minValue); + data.writeLong(gcd); + data.writeVInt(BLOCK_SIZE); + + final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + writer.add((nv.longValue() - minValue) / gcd); + } + writer.finish(); } else { meta.writeByte(DELTA_COMPRESSED); // delta-compressed @@ -222,7 +263,7 @@ @Override public void addSortedField(FieldInfo field, Iterable values, Iterable docToOrd) throws IOException { // write the ordinals as numerics - addNumericField(field, docToOrd); + addNumericField(field, docToOrd, false); // write the values as FST writeFST(field, values); Index: lucene/core/src/java/org/apache/lucene/util/MathUtil.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/MathUtil.java (révision 1469808) +++ lucene/core/src/java/org/apache/lucene/util/MathUtil.java (copie de travail) @@ -17,10 +17,11 @@ * limitations under the License. */ +import java.math.BigInteger; + /** * Math static utility methods. */ - public final class MathUtil { // No instance: @@ -42,4 +43,39 @@ } return ret; } + + /** Return the greatest common divisor of a and b, + * consistently with {@link BigInteger#gcd(BigInteger)}. + *

      NOTE: A greatest common divisor must be positive, but + * 2^64 cannot be expressed as a long although it + * is the GCD of {@link Long#MIN_VALUE} and 0 and the GCD of + * {@link Long#MIN_VALUE} and {@link Long#MIN_VALUE}. So in these 2 cases, + * and only them, this method will return {@link Long#MIN_VALUE}. */ + // see http://en.wikipedia.org/wiki/Binary_GCD_algorithm#Iterative_version_in_C.2B.2B_using_ctz_.28count_trailing_zeros.29 + public static long gcd(long a, long b) { + a = Math.abs(a); + b = Math.abs(b); + if (a == 0) { + return b; + } else if (b == 0) { + return a; + } + final int commonTrailingZeros = Long.numberOfTrailingZeros(a | b); + a >>>= Long.numberOfTrailingZeros(a); + while (true) { + b >>>= Long.numberOfTrailingZeros(b); + if (a == b) { + break; + } else if (a > b || a == Long.MIN_VALUE) { // MIN_VALUE is treated as 2^64 + final long tmp = a; + a = b; + b = tmp; + } + if (a == 1) { + break; + } + b -= a; + } + return a << commonTrailingZeros; + } } Index: lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java (révision 0) +++ lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java (copie de travail) @@ -0,0 +1,71 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +public class TestMathUtil extends LuceneTestCase { + + static long[] PRIMES = new long[] {2, 3, 5, 7, 11, 13, 17, 19, 23, 29}; + + static long randomLong() { + if (random().nextBoolean()) { + long l = 1; + if (random().nextBoolean()) { + l *= -1; + } + for (long i : PRIMES) { + final int m = random().nextInt(3); + for (int j = 0; j < m; ++j) { + l *= i; + } + } + return l; + } else if (random().nextBoolean()) { + return random().nextLong(); + } else { + return RandomPicks.randomFrom(random(), Arrays.asList(Long.MIN_VALUE, Long.MAX_VALUE, 0L, -1L, 1L)); + } + } + + // slow version used for testing + static long gcd(long l1, long l2) { + final BigInteger gcd = BigInteger.valueOf(l1).gcd(BigInteger.valueOf(l2)); + assert gcd.bitCount() <= 64; + return gcd.longValue(); + } + + public void testGCD() { + final int iters = atLeast(100); + for (int i = 0; i < iters; ++i) { + final long l1 = randomLong(); + final long l2 = randomLong(); + final long gcd = MathUtil.gcd(l1, l2); + final long actualGcd = gcd(l1, l2); + assertEquals(actualGcd, gcd); + if (gcd != 0) { + assertEquals(l1, (l1 / gcd) * gcd); + assertEquals(l2, (l2 / gcd) * gcd); + } + } + } + +} Modification de propriétés sur lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java (révision 1469808) +++ lucene/core/src/test/org/apache/lucene/codecs/lucene42/TestLucene42DocValuesFormat.java (copie de travail) @@ -18,12 +18,12 @@ */ import org.apache.lucene.codecs.Codec; -import org.apache.lucene.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase; /** * Tests Lucene42DocValuesFormat */ -public class TestLucene42DocValuesFormat extends BaseDocValuesFormatTestCase { +public class TestLucene42DocValuesFormat extends BaseCompressingDocValuesFormatTestCase { private final Codec codec = new Lucene42Codec(); @Override Index: lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java (révision 1469808) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java (copie de travail) @@ -17,6 +17,10 @@ * limitations under the License. */ +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.TABLE_COMPRESSED; + import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -37,6 +41,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.BlockPackedReader; import org.apache.lucene.util.packed.MonotonicBlockPackedReader; +import org.apache.lucene.util.packed.PackedInts; class CheapBastardDocValuesProducer extends DocValuesProducer { private final Map numerics; @@ -50,10 +55,11 @@ // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; + final int version; try { - CodecUtil.checkHeader(in, metaCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + version = CodecUtil.checkHeader(in, metaCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); numerics = new HashMap(); ords = new HashMap(); ordIndexes = new HashMap(); @@ -70,9 +76,12 @@ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); - CodecUtil.checkHeader(data, dataCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + final int version2 = CodecUtil.checkHeader(data, dataCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); + if (version != version2) { + throw new CorruptIndexException("Versions mismatch"); + } } private void readFields(IndexInput meta) throws IOException { @@ -140,6 +149,7 @@ static NumericEntry readNumericEntry(IndexInput meta) throws IOException { NumericEntry entry = new NumericEntry(); + entry.format = meta.readVInt(); entry.packedIntsVersion = meta.readVInt(); entry.offset = meta.readLong(); entry.count = meta.readVLong(); @@ -171,13 +181,48 @@ final IndexInput data = this.data.clone(); data.seek(entry.offset); - final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); - return new LongNumericDocValues() { - @Override - public long get(long id) { - return reader.get(id); - } - }; + switch (entry.format) { + case DELTA_COMPRESSED: + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return reader.get(id); + } + }; + case GCD_COMPRESSED: + final long min = data.readLong(); + final long mult = data.readLong(); + final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return min + mult * quotientReader.get(id); + } + }; + case TABLE_COMPRESSED: + final int uniqueValues = data.readVInt(); + if (uniqueValues > 256) { + throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values"); + } + final long[] values = new long[uniqueValues]; + for (int i = 0; i < uniqueValues; ++i) { + values[i] = data.readLong(); + } + final int bitsRequired = PackedInts.bitsRequired(uniqueValues - 1); + if (entry.count > Integer.MAX_VALUE) { + throw new CorruptIndexException("Cannot use TABLE_COMPRESSED with more than MAX_VALUE values"); + } + final PackedInts.Reader ords = PackedInts.getDirectReaderNoHeader(data, PackedInts.Format.PACKED, entry.packedIntsVersion, (int) entry.count, bitsRequired); + return new LongNumericDocValues() { + @Override + long get(long id) { + return values[(int) ords.get((int) id)]; + } + }; + default: + throw new CorruptIndexException("Unknown format: " + entry.format); + } } @Override @@ -315,6 +360,7 @@ static class NumericEntry { long offset; + int format; int packedIntsVersion; long count; int blockSize; Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseCompressingDocValuesFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseCompressingDocValuesFormatTestCase.java (révision 0) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseCompressingDocValuesFormatTestCase.java (copie de travail) @@ -0,0 +1,128 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.packed.PackedInts; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +/** Extends {@link BaseDocValuesFormatTestCase} to add compression checks. */ +public abstract class BaseCompressingDocValuesFormatTestCase extends BaseDocValuesFormatTestCase { + + static long dirSize(Directory d) throws IOException { + long size = 0; + for (String file : d.listAll()) { + size += d.fileLength(file); + } + return size; + } + + public void testUniqueValuesCompression() throws IOException { + final Directory dir = new RAMDirectory(); + final IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + final IndexWriter iwriter = new IndexWriter(dir, iwc); + + final int uniqueValueCount = _TestUtil.nextInt(random(), 1, 256); + final List values = new ArrayList(); + + final Document doc = new Document(); + final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0); + doc.add(dvf); + for (int i = 0; i < 300; ++i) { + final long value; + if (values.size() < uniqueValueCount) { + value = random().nextLong(); + values.add(value); + } else { + value = RandomPicks.randomFrom(random(), values); + } + dvf.setLongValue(value); + iwriter.addDocument(doc); + } + iwriter.forceMerge(1); + final long size1 = dirSize(dir); + for (int i = 0; i < 20; ++i) { + dvf.setLongValue(RandomPicks.randomFrom(random(), values)); + iwriter.addDocument(doc); + } + iwriter.forceMerge(1); + final long size2 = dirSize(dir); + // make sure the new longs did not cost 8 bytes each + assertTrue(size2 < size1 + 8 * 20); + } + + public void testDateCompression() throws IOException { + final Directory dir = new RAMDirectory(); + final IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + final IndexWriter iwriter = new IndexWriter(dir, iwc); + + final long base = 13; // prime + final long day = 1000L * 60 * 60 * 24; + + final Document doc = new Document(); + final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0); + doc.add(dvf); + for (int i = 0; i < 300; ++i) { + dvf.setLongValue(base + random().nextInt(1000) * day); + iwriter.addDocument(doc); + } + iwriter.forceMerge(1); + final long size1 = dirSize(dir); + for (int i = 0; i < 50; ++i) { + dvf.setLongValue(base + random().nextInt(1000) * day); + iwriter.addDocument(doc); + } + iwriter.forceMerge(1); + final long size2 = dirSize(dir); + // make sure the new longs costed less than if they had only been packed + assertTrue(size2 < size1 + (PackedInts.bitsRequired(day) * 50) / 8); + } + + public void testSingleBigValueCompression() throws IOException { + final Directory dir = new RAMDirectory(); + final IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + final IndexWriter iwriter = new IndexWriter(dir, iwc); + + final Document doc = new Document(); + final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0); + doc.add(dvf); + for (int i = 0; i < 20000; ++i) { + dvf.setLongValue(i & 1023); + iwriter.addDocument(doc); + } + iwriter.forceMerge(1); + final long size1 = dirSize(dir); + dvf.setLongValue(Long.MAX_VALUE); + iwriter.addDocument(doc); + iwriter.forceMerge(1); + final long size2 = dirSize(dir); + // make sure the new value did not grow the bpv for every other value + assertTrue(size2 < size1 + (20000 * (63 - 10)) / 8); + } + +} Modification de propriétés sur lucene/test-framework/src/java/org/apache/lucene/index/BaseCompressingDocValuesFormatTestCase.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (révision 1469808) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (copie de travail) @@ -25,15 +25,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.Map.Entry; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.lucene42.Lucene42Codec; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -44,7 +42,6 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInfo.DocValuesType; import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -1122,8 +1119,21 @@ w.close(); dir.close(); } - - private void doTestNumericsVsStoredFields(long minValue, long maxValue) throws Exception { + + static abstract class LongProducer { + abstract long next(); + } + + private void doTestNumericsVsStoredFields(final long minValue, final long maxValue) throws Exception { + doTestNumericsVsStoredFields(new LongProducer() { + @Override + long next() { + return _TestUtil.nextLong(random(), minValue, maxValue); + } + }); + } + + private void doTestNumericsVsStoredFields(LongProducer longs) throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); @@ -1137,9 +1147,12 @@ // index some docs int numDocs = atLeast(300); + // numDocs should be always > 256 so that in case of a codec that optimizes + // for numbers of values <= 256, all storage layouts are tested + assert numDocs > 256; for (int i = 0; i < numDocs; i++) { idField.setStringValue(Integer.toString(i)); - long value = _TestUtil.nextLong(random(), minValue, maxValue); + long value = longs.next(); storedField.setStringValue(Long.toString(value)); dvField.setLongValue(value); writer.addDocument(doc); @@ -1154,6 +1167,11 @@ int id = random().nextInt(numDocs); writer.deleteDocuments(new Term("id", Integer.toString(id))); } + + // merge some segments and ensure that at least one of them has more than + // 256 values + writer.forceMerge(numDocs / 256); + writer.close(); // compare @@ -2007,4 +2025,39 @@ doTestSortedSetVsUninvertedField(1, 10); } } + + public void testGCDCompression() throws Exception { + int numIterations = atLeast(1); + for (int i = 0; i < numIterations; i++) { + final long min = - (((long) random().nextInt(1 << 30)) << 32); + final long mul = random().nextInt() & 0xFFFFFFFFL; + final LongProducer longs = new LongProducer() { + @Override + long next() { + return min + mul * random().nextInt(1 << 20); + } + }; + doTestNumericsVsStoredFields(longs); + } + } + + public void testZeros() throws Exception { + doTestNumericsVsStoredFields(0, 0); + } + + public void testZeroOrMin() throws Exception { + // try to make GCD compression fail if the format did not anticipate that + // the GCD of 0 and MIN_VALUE is negative + int numIterations = atLeast(1); + for (int i = 0; i < numIterations; i++) { + final LongProducer longs = new LongProducer() { + @Override + long next() { + return random().nextBoolean() ? 0 : Long.MIN_VALUE; + } + }; + doTestNumericsVsStoredFields(longs); + } + } + } Index: lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java =================================================================== --- lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java (révision 1469808) +++ lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestDiskDocValuesFormat.java (copie de travail) @@ -18,13 +18,13 @@ */ import org.apache.lucene.codecs.Codec; -import org.apache.lucene.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase; import org.apache.lucene.util._TestUtil; /** * Tests DiskDocValuesFormat */ -public class TestDiskDocValuesFormat extends BaseDocValuesFormatTestCase { +public class TestDiskDocValuesFormat extends BaseCompressingDocValuesFormatTestCase { private final Codec codec = _TestUtil.alwaysDocValuesFormat(new DiskDocValuesFormat()); @Override Index: lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java =================================================================== --- lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java (révision 0) +++ lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java (copie de travail) @@ -0,0 +1,35 @@ +package org.apache.lucene.codecs.diskdv; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesFormat; +import org.apache.lucene.index.BaseCompressingDocValuesFormatTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * Tests CheapBastardDocValuesFormat + */ +public class TestCheapBastardDocValuesFormat extends BaseCompressingDocValuesFormatTestCase { + private final Codec codec = _TestUtil.alwaysDocValuesFormat(new CheapBastardDocValuesFormat()); + + @Override + protected Codec getCodec() { + return codec; + } +} Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java (révision 1469808) +++ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java (copie de travail) @@ -18,6 +18,8 @@ */ import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesConsumer; @@ -27,6 +29,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.MathUtil; import org.apache.lucene.util.packed.BlockPackedWriter; import org.apache.lucene.util.packed.MonotonicBlockPackedWriter; import org.apache.lucene.util.packed.PackedInts; @@ -36,6 +39,13 @@ static final int BLOCK_SIZE = 16384; + /** Compressed using packed blocks of ints. */ + public static final int DELTA_COMPRESSED = 0; + /** Compressed by computing the GCD. */ + public static final int GCD_COMPRESSED = 1; + /** Compressed by giving IDs to unique values. */ + public static final int TABLE_COMPRESSED = 2; + final IndexOutput data, meta; final int maxDoc; @@ -59,23 +69,105 @@ @Override public void addNumericField(FieldInfo field, Iterable values) throws IOException { + addNumericField(field, values, true); + } + + void addNumericField(FieldInfo field, Iterable values, boolean optimizeStorage) throws IOException { long count = 0; - for (@SuppressWarnings("unused") Number nv : values) { - ++count; + long minValue = Long.MAX_VALUE; + long maxValue = Long.MIN_VALUE; + long gcd = 0; + // TODO: more efficient? + HashSet uniqueValues = null; + if (optimizeStorage) { + uniqueValues = new HashSet<>(); + + for (Number nv : values) { + final long v = nv.longValue(); + + if (gcd != 1) { + if (v < - Long.MAX_VALUE / 2 || v > Long.MAX_VALUE / 2) { + // in that case v - minValue might overflow and make the GCD computation return + // wrong results. Since these extreme values are unlikely, we just discard + // GCD computation for them + gcd = 1; + } else if (count != 0) { // minValue needs to be set first + gcd = MathUtil.gcd(gcd, v - minValue); + } + } + + minValue = Math.min(minValue, v); + maxValue = Math.max(maxValue, v); + + if (uniqueValues != null) { + if (uniqueValues.add(v)) { + if (uniqueValues.size() > 256) { + uniqueValues = null; + } + } + } + + ++count; + } + } else { + for (@SuppressWarnings("unused") Number nv : values) { + ++count; + } } + final int format; + if (uniqueValues != null + && ((maxValue - minValue) < 0L || (maxValue - minValue) > 256) + && count <= Integer.MAX_VALUE) { + format = TABLE_COMPRESSED; + } else if (gcd != 0 && gcd != 1) { + format = GCD_COMPRESSED; + } else { + format = DELTA_COMPRESSED; + } meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.NUMERIC); + meta.writeVInt(format); meta.writeVInt(PackedInts.VERSION_CURRENT); meta.writeLong(data.getFilePointer()); meta.writeVLong(count); meta.writeVInt(BLOCK_SIZE); - final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); - for (Number nv : values) { - writer.add(nv.longValue()); + switch (format) { + case GCD_COMPRESSED: + data.writeLong(minValue); + data.writeLong(gcd); + final BlockPackedWriter quotientWriter = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + quotientWriter.add((nv.longValue() - minValue) / gcd); + } + quotientWriter.finish(); + break; + case DELTA_COMPRESSED: + final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + writer.add(nv.longValue()); + } + writer.finish(); + break; + case TABLE_COMPRESSED: + final Long[] decode = uniqueValues.toArray(new Long[uniqueValues.size()]); + final HashMap encode = new HashMap(); + data.writeVInt(decode.length); + for (int i = 0; i < decode.length; i++) { + data.writeLong(decode[i]); + encode.put(decode[i], i); + } + final int bitsRequired = PackedInts.bitsRequired(uniqueValues.size() - 1); + final PackedInts.Writer ordsWriter = PackedInts.getWriterNoHeader(data, PackedInts.Format.PACKED, (int) count, bitsRequired, PackedInts.DEFAULT_BUFFER_SIZE); + for (Number nv : values) { + ordsWriter.add(encode.get(nv.longValue())); + } + ordsWriter.finish(); + break; + default: + throw new AssertionError(); } - writer.finish(); } @Override @@ -120,7 +212,7 @@ meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.SORTED); addBinaryField(field, values); - addNumericField(field, docToOrd); + addNumericField(field, docToOrd, false); } @Override @@ -131,11 +223,12 @@ addBinaryField(field, values); // write the stream of ords as a numeric field // NOTE: we could return an iterator that delta-encodes these within a doc - addNumericField(field, ords); + addNumericField(field, ords, false); // write the doc -> ord count as a absolute index to the stream meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.NUMERIC); + meta.writeVInt(DELTA_COMPRESSED); meta.writeVInt(PackedInts.VERSION_CURRENT); meta.writeLong(data.getFilePointer()); meta.writeVLong(maxDoc); Index: lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (révision 1469808) +++ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (copie de travail) @@ -17,6 +17,10 @@ * limitations under the License. */ +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.TABLE_COMPRESSED; + import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -37,6 +41,7 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.packed.BlockPackedReader; import org.apache.lucene.util.packed.MonotonicBlockPackedReader; +import org.apache.lucene.util.packed.PackedInts; class DiskDocValuesProducer extends DocValuesProducer { private final Map numerics; @@ -55,10 +60,11 @@ // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; + final int version; try { - CodecUtil.checkHeader(in, metaCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + version = CodecUtil.checkHeader(in, metaCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); numerics = new HashMap(); ords = new HashMap(); ordIndexes = new HashMap(); @@ -75,9 +81,12 @@ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); - CodecUtil.checkHeader(data, dataCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + final int version2 = CodecUtil.checkHeader(data, dataCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); + if (version != version2) { + throw new CorruptIndexException("Format versions mismatch"); + } } private void readFields(IndexInput meta, FieldInfos infos) throws IOException { @@ -145,6 +154,7 @@ static NumericEntry readNumericEntry(IndexInput meta) throws IOException { NumericEntry entry = new NumericEntry(); + entry.format = meta.readVInt(); entry.packedIntsVersion = meta.readVInt(); entry.offset = meta.readLong(); entry.count = meta.readVLong(); @@ -176,13 +186,48 @@ final IndexInput data = this.data.clone(); data.seek(entry.offset); - final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); - return new LongNumericDocValues() { - @Override - public long get(long id) { - return reader.get(id); - } - }; + switch (entry.format) { + case DELTA_COMPRESSED: + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return reader.get(id); + } + }; + case GCD_COMPRESSED: + final long min = data.readLong(); + final long mult = data.readLong(); + final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return min + mult * quotientReader.get(id); + } + }; + case TABLE_COMPRESSED: + final int uniqueValues = data.readVInt(); + if (uniqueValues > 256) { + throw new CorruptIndexException("TABLE_COMPRESSED cannot have more than 256 distinct values"); + } + final long[] values = new long[uniqueValues]; + for (int i = 0; i < uniqueValues; ++i) { + values[i] = data.readLong(); + } + final int bitsRequired = PackedInts.bitsRequired(uniqueValues - 1); + if (entry.count > Integer.MAX_VALUE) { + throw new CorruptIndexException("Cannot use TABLE_COMPRESSED with more than MAX_VALUE values"); + } + final PackedInts.Reader ords = PackedInts.getDirectReaderNoHeader(data, PackedInts.Format.PACKED, entry.packedIntsVersion, (int) entry.count, bitsRequired); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return values[(int) ords.get((int) id)]; + } + }; + default: + throw new CorruptIndexException("Unknown format: " + entry.format); + } } @Override @@ -350,6 +395,7 @@ static class NumericEntry { long offset; + int format; int packedIntsVersion; long count; int blockSize; Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (révision 1469808) +++ lucene/CHANGES.txt (copie de travail) @@ -42,6 +42,13 @@ * LUCENE-4935: CustomScoreQuery wrongly applied its query boost twice (boost^2). (Robert Muir) +Optimizations + +* LUCENE-4936: Improve numeric doc values compression in case all values share + a common divisor. In particular, this improves the compression ratio of dates + without time when they are encoded as milliseconds since Epoch. + (Robert Muir, Adrien Grand) + ======================= Lucene 4.3.0 ======================= Changes in backwards compatibility policy