Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1552546) +++ lucene/CHANGES.txt (working copy) @@ -104,6 +104,9 @@ * LUCENE-5285: Improved highlighting of multi-valued fields with FastVectorHighlighter. (Nik Everett via Adrien Grand) +* LUCENE-5373: Memory usage of [Lucene42/Memory/Direct]DocValuesFormat was + over-estimated. (Shay Banon, Adrien Grand, Robert Muir) + Changes in Runtime Behavior * LUCENE-5362: IndexReader and SegmentCoreReaders now throw Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (revision 1552546) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectDocValuesProducer.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; @@ -62,6 +63,7 @@ private final Map docsWithFieldInstances = new HashMap(); private final int maxDoc; + private final AtomicLong ramBytesUsed; static final byte NUMBER = 0; static final byte BYTES = 1; @@ -76,6 +78,7 @@ String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension); // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); + ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); boolean success = false; final int version; try { @@ -178,8 +181,7 @@ @Override public long ramBytesUsed() { - // TODO: optimize me - return RamUsageEstimator.sizeOf(this); + return ramBytesUsed.get(); } @Override @@ -199,9 +201,8 @@ case 1: { final byte[] values = new byte[entry.count]; - for(int i=0;i docsWithFieldInstances = new HashMap(); private final int maxDoc; + private final AtomicLong ramBytesUsed; - static final byte NUMBER = 0; static final byte BYTES = 1; static final byte FST = 2; @@ -107,7 +108,7 @@ binaries = new HashMap(); fsts = new HashMap(); readFields(in, state.fieldInfos); - + ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); success = true; } finally { if (success) { @@ -204,8 +205,7 @@ @Override public long ramBytesUsed() { - // TODO: optimize me - return RamUsageEstimator.sizeOf(this); + return ramBytesUsed.get(); } private NumericDocValues loadNumeric(FieldInfo field) throws IOException { @@ -224,6 +224,7 @@ final int formatID = data.readVInt(); final int bitsPerValue = data.readVInt(); final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); + ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); return new NumericDocValues() { @Override public long get(int docID) { @@ -233,10 +234,12 @@ case DELTA_COMPRESSED: final int blockSize = data.readVInt(); final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); + ramBytesUsed.addAndGet(reader.ramBytesUsed()); return reader; case UNCOMPRESSED: final byte bytes[] = new byte[maxDoc]; data.readBytes(bytes, 0, bytes.length); + ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes)); return new NumericDocValues() { @Override public long get(int docID) { @@ -248,6 +251,7 @@ final long mult = data.readLong(); final int quotientBlockSize = data.readVInt(); final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); + ramBytesUsed.addAndGet(quotientReader.ramBytesUsed()); return new NumericDocValues() { @Override public long get(int docID) { @@ -277,6 +281,7 @@ final PagedBytes.Reader bytesReader = bytes.freeze(true); if (entry.minLength == entry.maxLength) { final int fixedLength = entry.minLength; + ramBytesUsed.addAndGet(bytes.ramBytesUsed()); return new BinaryDocValues() { @Override public void get(int docID, BytesRef result) { @@ -286,6 +291,7 @@ } else { data.seek(data.getFilePointer() + entry.missingBytes); final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); + ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed()); return new BinaryDocValues() { @Override public void get(int docID, BytesRef result) { @@ -309,6 +315,7 @@ if (instance == null) { data.seek(entry.offset); instance = new FST(data, PositiveIntOutputs.getSingleton()); + ramBytesUsed.addAndGet(instance.sizeInBytes()); fstInstances.put(field.number, instance); } } @@ -383,6 +390,7 @@ if (instance == null) { data.seek(entry.offset); instance = new FST(data, PositiveIntOutputs.getSingleton()); + ramBytesUsed.addAndGet(instance.sizeInBytes()); fstInstances.put(field.number, instance); } } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (revision 1552546) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; @@ -73,8 +74,8 @@ new HashMap>(); private final int maxDoc; + private final AtomicLong ramBytesUsed; - static final byte NUMBER = 0; static final byte BYTES = 1; static final byte FST = 2; @@ -96,6 +97,7 @@ // read in the entries from the metadata file. IndexInput in = state.directory.openInput(metaName, state.context); boolean success = false; + ramBytesUsed = new AtomicLong(RamUsageEstimator.shallowSizeOfInstance(getClass())); final int version; try { version = CodecUtil.checkHeader(in, metaCodec, @@ -190,7 +192,7 @@ @Override public long ramBytesUsed() { - return RamUsageEstimator.sizeOf(this); + return ramBytesUsed.get(); } private NumericDocValues loadNumeric(FieldInfo field) throws IOException { @@ -209,6 +211,7 @@ final int formatID = data.readVInt(); final int bitsPerValue = data.readVInt(); final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue); + ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed()); return new NumericDocValues() { @Override public long get(int docID) { @@ -218,15 +221,12 @@ case DELTA_COMPRESSED: final int blockSize = data.readVInt(); final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false); - return new NumericDocValues() { - @Override - public long get(int docID) { - return reader.get(docID); - } - }; + ramBytesUsed.addAndGet(reader.ramBytesUsed()); + return reader; case UNCOMPRESSED: final byte bytes[] = new byte[maxDoc]; data.readBytes(bytes, 0, bytes.length); + ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes)); return new NumericDocValues() { @Override public long get(int docID) { @@ -238,6 +238,7 @@ final long mult = data.readLong(); final int quotientBlockSize = data.readVInt(); final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false); + ramBytesUsed.addAndGet(quotientReader.ramBytesUsed()); return new NumericDocValues() { @Override public long get(int docID) { @@ -267,6 +268,7 @@ final PagedBytes.Reader bytesReader = bytes.freeze(true); if (entry.minLength == entry.maxLength) { final int fixedLength = entry.minLength; + ramBytesUsed.addAndGet(bytes.ramBytesUsed()); return new BinaryDocValues() { @Override public void get(int docID, BytesRef result) { @@ -275,6 +277,7 @@ }; } else { final MonotonicBlockPackedReader addresses = new MonotonicBlockPackedReader(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false); + ramBytesUsed.addAndGet(bytes.ramBytesUsed() + addresses.ramBytesUsed()); return new BinaryDocValues() { @Override public void get(int docID, BytesRef result) { @@ -295,6 +298,7 @@ if (instance == null) { data.seek(entry.offset); instance = new FST(data, PositiveIntOutputs.getSingleton()); + ramBytesUsed.addAndGet(instance.sizeInBytes()); fstInstances.put(field.number, instance); } } @@ -369,6 +373,7 @@ if (instance == null) { data.seek(entry.offset); instance = new FST(data, PositiveIntOutputs.getSingleton()); + ramBytesUsed.addAndGet(instance.sizeInBytes()); fstInstances.put(field.number, instance); } }