diff --git oak-doc/src/site/markdown/nodestore/segment/records.md oak-doc/src/site/markdown/nodestore/segment/records.md index 57c4eabf18..24a25805be 100644 --- oak-doc/src/site/markdown/nodestore/segment/records.md +++ oak-doc/src/site/markdown/nodestore/segment/records.md @@ -292,6 +292,8 @@ only a "diff" of the map is stored. This prevents the full storage of the modified map, which can save a considerable amount of space if the original map was big. +_Warning: A map record can store up to 2^29 - 1 (i.e. 536.870.911) entries! In order to avoid reaching this number and possibly running into issues from surpassing it, log messages are printed after reaching 400.000.000 entries and writing beyond 500.000.000 entries is not allowed unless the boolean system property `oak.segmentNodeStore.allowWritesOnHugeMapRecord` is set. Finally, the segment store does not allow writing map records with more than 536.000.000 entries._ + ### Template records A template record stores metadata about nodes that, on average, don't change so diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java index 7bb3c11237..f4fcc0259f 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java @@ -221,6 +221,8 @@ public class DefaultSegmentWriter implements SegmentWriter { private final Cache nodeCache; + private long ts; + SegmentWriteOperation(@NotNull GCGeneration gcGeneration) { int generation = gcGeneration.getGeneration(); this.gcGeneration = gcGeneration; @@ -233,10 +235,46 @@ public class DefaultSegmentWriter implements SegmentWriter { return writer -> recordWriter.write(writer, store); } - private RecordId writeMap(@Nullable MapRecord base, - @NotNull Map changes - ) - throws IOException { + private boolean shouldLog() { + long ts = System.currentTimeMillis(); + if ((ts - this.ts) / 1000 >= 1) { + this.ts = ts; + return true; + } else { + return false; + } + } + + private RecordId writeMap(@Nullable MapRecord base, @NotNull Map changes) throws IOException { + if (base != null) { + if (base.size() >= MapRecord.ERROR_SIZE_HARD_STOP) { + throw new UnsupportedOperationException("Map record has more than " + MapRecord.ERROR_SIZE_HARD_STOP + " direct entries. Writing is not allowed. Please remove entries."); + } else if (base.size() >= MapRecord.ERROR_SIZE_DISCARD_WRITES) { + if (!Boolean.getBoolean("oak.segmentNodeStore.allowWritesOnHugeMapRecord")) { + if (shouldLog()) { + LOG.error( + "Map entry has more than {} entries. Writing more than {} entries (up to the hard limit of {}) is only allowed " + + "if the system property \"oak.segmentNodeStore.allowWritesOnHugeMapRecord\" is set", + MapRecord.ERROR_SIZE, MapRecord.ERROR_SIZE_DISCARD_WRITES, MapRecord.ERROR_SIZE_HARD_STOP); + } + + // discard changes and return the previous map record id + return base.getRecordId(); + } + } else if (base.size() >= MapRecord.ERROR_SIZE) { + if (shouldLog()) { + LOG.error( + "Map entry has more than {} entries. Writing more than {} entries (up to the hard limit of {}) is only allowed " + + "if the system property \"oak.segmentNodeStore.allowWritesOnHugeMapRecord\" is set", + MapRecord.ERROR_SIZE, MapRecord.ERROR_SIZE_DISCARD_WRITES, MapRecord.ERROR_SIZE_HARD_STOP); + } + } else if (base.size() >= MapRecord.WARN_SIZE) { + if (shouldLog()) { + LOG.warn("Map entry has more than {} entries. Please remove entries.", MapRecord.WARN_SIZE); + } + } + } + if (base != null && base.isDiff()) { Segment segment = base.getSegment(); RecordId key = segment.readRecordId(base.getRecordNumber(), 8); @@ -299,6 +337,7 @@ public class DefaultSegmentWriter implements SegmentWriter { } private RecordId writeMapBranch(int level, int size, MapRecord... buckets) throws IOException { + checkElementIndex(size, MapRecord.MAX_SIZE); int bitmap = 0; List bucketIds = newArrayListWithCapacity(buckets.length); for (int i = 0; i < buckets.length; i++) { diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java index 37e78b1f0a..9f534a02e7 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/MapRecord.java @@ -97,6 +97,28 @@ public class MapRecord extends Record { */ protected static final int MAX_SIZE = (1 << SIZE_BITS) - 1; // ~268e6 + /** + * Going over this limit will generate a warning message in the log + */ + protected static final int WARN_SIZE = 400_000_000; + + /** + * Going over this limit will generate an error message in the log + */ + protected static final int ERROR_SIZE = 450_000_000; + + /** + * Going over this limit will generate an error message in the log and won't allow writes + * unless oak.segmentNodeStore.allowWritesOnHugeMapRecord system property is set. + */ + protected static final int ERROR_SIZE_DISCARD_WRITES = 500_000_000; + + /** + * Going over this limit will generate an error message in the log and won't allow any writes + * whatsoever. Moreover {@link UnsupportedOperationException} will be thrown. + */ + protected static final int ERROR_SIZE_HARD_STOP = 536_000_000; + MapRecord(@NotNull SegmentReader reader, @NotNull RecordId id) { super(id); this.reader = checkNotNull(reader); @@ -180,7 +202,7 @@ public class MapRecord extends Record { int level = getLevel(head); if (isBranch(size, level)) { // this is an intermediate branch record - // check if a matching bucket exists, and recurse + // check if a matching bucket exists, and recurse int bitmap = segment.readInt(getRecordNumber(), 4); int mask = (1 << BITS_PER_LEVEL) - 1; int shift = 32 - (level + 1) * BITS_PER_LEVEL;