Index: lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java (révision 1469372) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/cheapbastard/CheapBastardDocValuesProducer.java (copie de travail) @@ -17,6 +17,10 @@ * limitations under the License. */ +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesFormat.VERSION_GCD_COMPRESSION; + import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -44,6 +48,7 @@ private final Map ordIndexes; private final Map binaries; private final IndexInput data; + private final int version; CheapBastardDocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException { String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); @@ -51,9 +56,9 @@ IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; try { - CodecUtil.checkHeader(in, metaCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + version = CodecUtil.checkHeader(in, metaCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); numerics = new HashMap(); ords = new HashMap(); ordIndexes = new HashMap(); @@ -72,7 +77,7 @@ data = state.directory.openInput(dataName, state.context); CodecUtil.checkHeader(data, dataCodec, DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + DiskDocValuesFormat.VERSION_CURRENT); } private void readFields(IndexInput meta) throws IOException { @@ -138,8 +143,13 @@ } } - static NumericEntry readNumericEntry(IndexInput meta) throws IOException { + NumericEntry readNumericEntry(IndexInput meta) throws IOException { NumericEntry entry = new NumericEntry(); + if (version >= VERSION_GCD_COMPRESSION) { + entry.format = meta.readVInt(); + } else { + entry.format = DELTA_COMPRESSED; + } entry.packedIntsVersion = meta.readVInt(); entry.offset = meta.readLong(); entry.count = meta.readVLong(); @@ -171,13 +181,27 @@ final IndexInput data = this.data.clone(); data.seek(entry.offset); - final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); - return new LongNumericDocValues() { - @Override - public long get(long id) { - return reader.get(id); - } - }; + switch (entry.format) { + case DELTA_COMPRESSED: + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return reader.get(id); + } + }; + case GCD_COMPRESSED: + final long mul = data.readVLong(); + final BlockPackedReader divReader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return mul * divReader.get(id); + } + }; + default: + throw new CorruptIndexException("Unknown format: " + entry.format); + } } @Override @@ -315,6 +339,7 @@ static class NumericEntry { long offset; + int format; int packedIntsVersion; long count; int blockSize; Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (révision 1469372) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (copie de travail) @@ -25,15 +25,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; -import java.util.Map.Entry; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.DocValuesFormat; -import org.apache.lucene.codecs.lucene42.Lucene42Codec; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -44,7 +42,6 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.FieldInfo.DocValuesType; import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -1122,8 +1119,21 @@ w.close(); dir.close(); } - - private void doTestNumericsVsStoredFields(long minValue, long maxValue) throws Exception { + + static abstract class LongProducer { + abstract long next(); + } + + private void doTestNumericsVsStoredFields(final long minValue, final long maxValue) throws Exception { + doTestNumericsVsStoredFields(new LongProducer() { + @Override + long next() { + return _TestUtil.nextLong(random(), minValue, maxValue); + } + }); + } + + private void doTestNumericsVsStoredFields(LongProducer longs) throws Exception { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf); @@ -1137,9 +1147,12 @@ // index some docs int numDocs = atLeast(300); + // numDocs should be always > 256 so that in case of a codec that optimizes + // for numbers of values <= 256, all storage layouts are tested + assert numDocs > 256; for (int i = 0; i < numDocs; i++) { idField.setStringValue(Integer.toString(i)); - long value = _TestUtil.nextLong(random(), minValue, maxValue); + long value = longs.next(); storedField.setStringValue(Long.toString(value)); dvField.setLongValue(value); writer.addDocument(doc); @@ -1154,6 +1167,11 @@ int id = random().nextInt(numDocs); writer.deleteDocuments(new Term("id", Integer.toString(id))); } + + // merge some segments and ensure that at least one of them has more than + // 256 values + writer.forceMerge(numDocs / 256); + writer.close(); // compare @@ -2007,4 +2025,38 @@ doTestSortedSetVsUninvertedField(1, 10); } } + + public void testGCDCompression() throws Exception { + int numIterations = atLeast(1); + for (int i = 0; i < numIterations; i++) { + final long mul = random().nextInt() & 0xFFFFFFFFL; + final LongProducer longs = new LongProducer() { + @Override + long next() { + return mul * random().nextInt(1 << 20); + } + }; + doTestNumericsVsStoredFields(longs); + } + } + + public void testZeros() throws Exception { + doTestNumericsVsStoredFields(0, 0); + } + + public void testZeroOrMin() throws Exception { + // try to make GCD compression fail if the format did not anticipate that + // the GCD of 0 and MIN_VALUE is negative + int numIterations = atLeast(1); + for (int i = 0; i < numIterations; i++) { + final LongProducer longs = new LongProducer() { + @Override + long next() { + return random().nextBoolean() ? 0 : Long.MIN_VALUE; + } + }; + doTestNumericsVsStoredFields(longs); + } + } + } Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (révision 1469372) +++ lucene/CHANGES.txt (copie de travail) @@ -42,6 +42,13 @@ * LUCENE-4935: CustomScoreQuery wrongly applied its query boost twice (boost^2). (Robert Muir) +Optimizations + +* LUCENE-4936: Improve numeric doc values compression in case all values share + a common divisor. In particular, this improves the compression ratio of dates + without time when they are encoded as milliseconds since Epoch. + (Robert Muir, Adrien Grand) + ======================= Lucene 4.3.0 ======================= Changes in backwards compatibility policy Index: lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java (révision 1469372) +++ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesFormat.java (copie de travail) @@ -54,7 +54,8 @@ public static final String META_CODEC = "DiskDocValuesMetadata"; public static final String META_EXTENSION = "dvdm"; public static final int VERSION_START = 0; - public static final int VERSION_CURRENT = VERSION_START; + public static final int VERSION_GCD_COMPRESSION = 1; + public static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION; public static final byte NUMERIC = 0; public static final byte BINARY = 1; public static final byte SORTED = 2; Index: lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java (révision 1469372) +++ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesConsumer.java (copie de travail) @@ -27,6 +27,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.MathUtil; import org.apache.lucene.util.packed.BlockPackedWriter; import org.apache.lucene.util.packed.MonotonicBlockPackedWriter; import org.apache.lucene.util.packed.PackedInts; @@ -36,6 +37,11 @@ static final int BLOCK_SIZE = 16384; + /** Compressed using packed blocks of ints. */ + public static final int DELTA_COMPRESSED = 0; + /** Compressed by computing the GCD. */ + public static final int GCD_COMPRESSED = 1; + final IndexOutput data, meta; final int maxDoc; @@ -59,23 +65,49 @@ @Override public void addNumericField(FieldInfo field, Iterable values) throws IOException { + addNumericField(field, values, true); + } + + void addNumericField(FieldInfo field, Iterable values, boolean optimizeStorage) throws IOException { long count = 0; - for (@SuppressWarnings("unused") Number nv : values) { + long gcd = 0; + for (Number nv : values) { ++count; + if (optimizeStorage && gcd != 1) { + gcd = MathUtil.gcd(nv.longValue(), gcd); + } } + final int format = gcd == 0 + || gcd == Long.MIN_VALUE // this is very unlikely, and handling this + // case would prevent us from using writeVLong + // since it only supports positive longs + || gcd == 1 + ? DELTA_COMPRESSED + : GCD_COMPRESSED; meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.NUMERIC); + meta.writeVInt(format); meta.writeVInt(PackedInts.VERSION_CURRENT); meta.writeLong(data.getFilePointer()); meta.writeVLong(count); meta.writeVInt(BLOCK_SIZE); - final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); - for (Number nv : values) { - writer.add(nv.longValue()); + if (format == GCD_COMPRESSED) { + data.writeVLong(gcd); + final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + writer.add(nv.longValue() / gcd); + } + writer.finish(); + } else { + final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + writer.add(nv.longValue()); + } + writer.finish(); } - writer.finish(); + } @Override @@ -120,7 +152,7 @@ meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.SORTED); addBinaryField(field, values); - addNumericField(field, docToOrd); + addNumericField(field, docToOrd, false); } @Override @@ -131,11 +163,12 @@ addBinaryField(field, values); // write the stream of ords as a numeric field // NOTE: we could return an iterator that delta-encodes these within a doc - addNumericField(field, ords); + addNumericField(field, ords, false); // write the doc -> ord count as a absolute index to the stream meta.writeVInt(field.number); meta.writeByte(DiskDocValuesFormat.NUMERIC); + meta.writeVInt(DELTA_COMPRESSED); meta.writeVInt(PackedInts.VERSION_CURRENT); meta.writeLong(data.getFilePointer()); meta.writeVLong(maxDoc); Index: lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (révision 1469372) +++ lucene/codecs/src/java/org/apache/lucene/codecs/diskdv/DiskDocValuesProducer.java (copie de travail) @@ -17,6 +17,10 @@ * limitations under the License. */ +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.diskdv.DiskDocValuesFormat.VERSION_GCD_COMPRESSION; + import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -44,6 +48,7 @@ private final Map ords; private final Map ordIndexes; private final IndexInput data; + private final int version; // memory-resident structures private final Map ordinalInstances = new HashMap(); @@ -56,9 +61,9 @@ IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; try { - CodecUtil.checkHeader(in, metaCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + version = CodecUtil.checkHeader(in, metaCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); numerics = new HashMap(); ords = new HashMap(); ordIndexes = new HashMap(); @@ -75,9 +80,12 @@ String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension); data = state.directory.openInput(dataName, state.context); - CodecUtil.checkHeader(data, dataCodec, - DiskDocValuesFormat.VERSION_START, - DiskDocValuesFormat.VERSION_START); + final int version2 = CodecUtil.checkHeader(data, dataCodec, + DiskDocValuesFormat.VERSION_START, + DiskDocValuesFormat.VERSION_CURRENT); + if (version != version2) { + throw new CorruptIndexException("Format versions mismatch"); + } } private void readFields(IndexInput meta, FieldInfos infos) throws IOException { @@ -143,8 +151,13 @@ } } - static NumericEntry readNumericEntry(IndexInput meta) throws IOException { + NumericEntry readNumericEntry(IndexInput meta) throws IOException { NumericEntry entry = new NumericEntry(); + if (version >= VERSION_GCD_COMPRESSION) { + entry.format = meta.readVInt(); + } else { + entry.format = DELTA_COMPRESSED; + } entry.packedIntsVersion = meta.readVInt(); entry.offset = meta.readLong(); entry.count = meta.readVLong(); @@ -176,13 +189,27 @@ final IndexInput data = this.data.clone(); data.seek(entry.offset); - final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); - return new LongNumericDocValues() { - @Override - public long get(long id) { - return reader.get(id); - } - }; + switch (entry.format) { + case DELTA_COMPRESSED: + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return reader.get(id); + } + }; + case GCD_COMPRESSED: + final long mul = data.readVLong(); + final BlockPackedReader divReader = new BlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, entry.count, true); + return new LongNumericDocValues() { + @Override + public long get(long id) { + return mul * divReader.get(id); + } + }; + default: + throw new CorruptIndexException("Unknown format: " + entry.format); + } } @Override @@ -350,6 +377,7 @@ static class NumericEntry { long offset; + int format; int packedIntsVersion; long count; int blockSize; Index: lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java =================================================================== --- lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java (révision 0) +++ lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java (copie de travail) @@ -0,0 +1,35 @@ +package org.apache.lucene.codecs.diskdv; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.cheapbastard.CheapBastardDocValuesFormat; +import org.apache.lucene.index.BaseDocValuesFormatTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * Tests CheapBastardDocValuesFormat + */ +public class TestCheapBastardDocValuesFormat extends BaseDocValuesFormatTestCase { + private final Codec codec = _TestUtil.alwaysDocValuesFormat(new CheapBastardDocValuesFormat()); + + @Override + protected Codec getCodec() { + return codec; + } +} Modification de propriétés sur lucene/codecs/src/test/org/apache/lucene/codecs/diskdv/TestCheapBastardDocValuesFormat.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java (révision 0) +++ lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java (copie de travail) @@ -0,0 +1,71 @@ +package org.apache.lucene.util; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +public class TestMathUtil extends LuceneTestCase { + + static long[] PRIMES = new long[] {2, 3, 5, 7, 11, 13, 17, 19, 23, 29}; + + static long randomLong() { + if (random().nextBoolean()) { + long l = 1; + if (random().nextBoolean()) { + l *= -1; + } + for (long i : PRIMES) { + final int m = random().nextInt(3); + for (int j = 0; j < m; ++j) { + l *= i; + } + } + return l; + } else if (random().nextBoolean()) { + return random().nextLong(); + } else { + return RandomPicks.randomFrom(random(), Arrays.asList(Long.MIN_VALUE, Long.MAX_VALUE, 0L, -1L, 1L)); + } + } + + // slow version used for testing + static long gcd(long l1, long l2) { + final BigInteger gcd = BigInteger.valueOf(l1).gcd(BigInteger.valueOf(l2)); + assert gcd.bitCount() <= 64; + return gcd.longValue(); + } + + public void testGCD() { + final int iters = atLeast(100); + for (int i = 0; i < iters; ++i) { + final long l1 = randomLong(); + final long l2 = randomLong(); + final long gcd = MathUtil.gcd(l1, l2); + final long actualGcd = gcd(l1, l2); + assertEquals(actualGcd, gcd); + if (gcd != 0) { + assertEquals(l1, (l1 / gcd) * gcd); + assertEquals(l2, (l2 / gcd) * gcd); + } + } + } + +} Modification de propriétés sur lucene/core/src/test/org/apache/lucene/util/TestMathUtil.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (révision 1469372) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (copie de travail) @@ -83,7 +83,7 @@ try { CodecUtil.checkHeader(in, metaCodec, Lucene42DocValuesConsumer.VERSION_START, - Lucene42DocValuesConsumer.VERSION_START); + Lucene42DocValuesConsumer.VERSION_CURRENT); numerics = new HashMap(); binaries = new HashMap(); fsts = new HashMap(); @@ -101,7 +101,7 @@ data = state.directory.openInput(dataName, state.context); CodecUtil.checkHeader(data, dataCodec, Lucene42DocValuesConsumer.VERSION_START, - Lucene42DocValuesConsumer.VERSION_START); + Lucene42DocValuesConsumer.VERSION_CURRENT); } private void readFields(IndexInput meta, FieldInfos infos) throws IOException { @@ -185,6 +185,16 @@ return bytes[docID]; } }; + } else if (entry.format == Lucene42DocValuesConsumer.GCD_COMPRESSED) { + final long mult = data.readVLong(); + final int blockSize = data.readVInt(); + final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); + return new NumericDocValues() { + @Override + public long get(int docID) { + return mult * reader.get(docID); + } + }; } else { throw new IllegalStateException(); } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (révision 1469372) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesConsumer.java (copie de travail) @@ -34,6 +34,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.MathUtil; import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.FST.INPUT_TYPE; @@ -49,7 +50,8 @@ */ class Lucene42DocValuesConsumer extends DocValuesConsumer { static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; + static final int VERSION_GCD_COMPRESSION = 1; + static final int VERSION_CURRENT = VERSION_GCD_COMPRESSION; static final byte NUMBER = 0; static final byte BYTES = 1; @@ -60,6 +62,7 @@ static final byte DELTA_COMPRESSED = 0; static final byte TABLE_COMPRESSED = 1; static final byte UNCOMPRESSED = 2; + static final byte GCD_COMPRESSED = 3; final IndexOutput data, meta; final int maxDoc; @@ -83,26 +86,37 @@ } } } - + @Override public void addNumericField(FieldInfo field, Iterable values) throws IOException { + addNumericField(field, values, true); + } + + void addNumericField(FieldInfo field, Iterable values, boolean optimizeStorage) throws IOException { meta.writeVInt(field.number); meta.writeByte(NUMBER); meta.writeLong(data.getFilePointer()); long minValue = Long.MAX_VALUE; long maxValue = Long.MIN_VALUE; + long gcd = 0; // TODO: more efficient? - HashSet uniqueValues = new HashSet(); - for(Number nv : values) { - long v = nv.longValue(); - minValue = Math.min(minValue, v); - maxValue = Math.max(maxValue, v); - if (uniqueValues != null) { - if (uniqueValues.add(v)) { - if (uniqueValues.size() > 256) { - uniqueValues = null; + HashSet uniqueValues = null; + if (optimizeStorage) { + uniqueValues = new HashSet<>(); + for(Number nv : values) { + long v = nv.longValue(); + minValue = Math.min(minValue, v); + maxValue = Math.max(maxValue, v); + if (uniqueValues != null) { + if (uniqueValues.add(v)) { + if (uniqueValues.size() > 256) { + uniqueValues = null; + } } } + if (gcd != 1) { + gcd = MathUtil.gcd(gcd, v); + } } } @@ -135,6 +149,18 @@ } writer.finish(); } + } else if (gcd != 0 && gcd != 1 && gcd != Long.MIN_VALUE) { + // no Long.MIN_VALE because this case is very unlikely and writeVLong does not handle negative numbers + meta.writeByte(GCD_COMPRESSED); + meta.writeVInt(PackedInts.VERSION_CURRENT); + data.writeVLong(gcd); + data.writeVInt(BLOCK_SIZE); + + final BlockPackedWriter writer = new BlockPackedWriter(data, BLOCK_SIZE); + for (Number nv : values) { + writer.add(nv.longValue() / gcd); + } + writer.finish(); } else { meta.writeByte(DELTA_COMPRESSED); // delta-compressed @@ -222,7 +248,7 @@ @Override public void addSortedField(FieldInfo field, Iterable values, Iterable docToOrd) throws IOException { // write the ordinals as numerics - addNumericField(field, docToOrd); + addNumericField(field, docToOrd, false); // write the values as FST writeFST(field, values); Index: lucene/core/src/java/org/apache/lucene/util/MathUtil.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/MathUtil.java (révision 1469372) +++ lucene/core/src/java/org/apache/lucene/util/MathUtil.java (copie de travail) @@ -17,10 +17,11 @@ * limitations under the License. */ +import java.math.BigInteger; + /** * Math static utility methods. */ - public final class MathUtil { // No instance: @@ -42,4 +43,39 @@ } return ret; } + + /** Return the greatest common divisor of a and b, + * consistently with {@link BigInteger#gcd(BigInteger)}. + *

NOTE: A greatest common divisor must be positive, but + * 2^64 cannot be expressed as a long although it + * is the GCD of {@link Long#MIN_VALUE} and 0 and the GCD of + * {@link Long#MIN_VALUE} and {@link Long#MIN_VALUE}. So in these 2 cases, + * and only them, this method will return {@link Long#MIN_VALUE}. */ + // see http://en.wikipedia.org/wiki/Binary_GCD_algorithm#Iterative_version_in_C.2B.2B_using_ctz_.28count_trailing_zeros.29 + public static long gcd(long a, long b) { + a = Math.abs(a); + b = Math.abs(b); + if (a == 0) { + return b; + } else if (b == 0) { + return a; + } + final int commonTrailingZeros = Long.numberOfTrailingZeros(a | b); + a >>>= Long.numberOfTrailingZeros(a); + while (true) { + b >>>= Long.numberOfTrailingZeros(b); + if (a == b) { + break; + } else if (a > b || a == Long.MIN_VALUE) { // MIN_VALUE is treated as 2^64 + final long tmp = a; + a = b; + b = tmp; + } + if (a == 1) { + break; + } + b -= a; + } + return a << commonTrailingZeros; + } }