Index: src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java (working copy) @@ -39,6 +39,7 @@ private long missCount; private long loadCount; private long evictionCount; + private long maxSize; /** * Add a mapping from {@code key} to {@code value}. Any existing mapping is replaced. @@ -57,6 +58,13 @@ public abstract long size(); /** + * @return max number of mappings + */ + public long getMaxSize() { + return maxSize; + } + + /** * @return access statistics for this cache */ public CacheStats getStats() { @@ -142,6 +150,7 @@ return remove; } }; + super.maxSize = size; } @Override Index: src/main/java/org/apache/jackrabbit/oak/segment/RecordCacheStats.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/RecordCacheStats.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/RecordCacheStats.java (working copy) @@ -47,14 +47,24 @@ @Nonnull private final Supplier elementCount; + @Nonnull + private final Supplier maxSize; + + private final long avgWeight; + private CacheStats lastSnapshot; - public RecordCacheStats( - @Nonnull String name, @Nonnull Supplier stats, @Nonnull Supplier elementCount) { + public RecordCacheStats(@Nonnull String name, + @Nonnull Supplier stats, + @Nonnull Supplier elementCount, + @Nonnull Supplier maxSize, + long avgWeight) { super(CacheStatsMBean.class); this.name = checkNotNull(name); this.stats = checkNotNull(stats); this.elementCount = checkNotNull(elementCount); + this.maxSize = checkNotNull(maxSize); + this.avgWeight = avgWeight; this.lastSnapshot = stats.get(); } @@ -140,11 +150,17 @@ @Override public long getMaxTotalWeight() { + if (avgWeight > 0) { + return maxSize.get() * avgWeight; + } return -1; } @Override public long estimateCurrentWeight() { + if (avgWeight > 0) { + return elementCount.get() * avgWeight; + } return -1; } Index: src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java (working copy) @@ -36,6 +36,7 @@ import com.google.common.base.Function; import com.google.common.base.Predicate; import com.google.common.base.Supplier; +import com.google.common.base.Suppliers; import com.google.common.cache.CacheStats; import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; import org.apache.jackrabbit.oak.segment.file.PriorityCache; @@ -295,15 +296,25 @@ @CheckForNull @Override public CacheStatsMBean getStringCacheStats() { + // TODO avg string entry + long avgStringWeight = Segment.MEDIUM_LIMIT * 9 / 10; return new RecordCacheStats("String deduplication cache stats", - accumulateRecordCacheStats(stringCaches), accumulateRecordCacheSizes(stringCaches)); + accumulateRecordCacheStats(stringCaches), + accumulateRecordCacheSizes(stringCaches), + accumulateRecordCacheMaxSizes(stringCaches), + avgStringWeight); } @CheckForNull @Override public CacheStatsMBean getTemplateCacheStats() { + // TODO avg template entry + long avgTemplate = 250; return new RecordCacheStats("Template deduplication cache stats", - accumulateRecordCacheStats(templateCaches), accumulateRecordCacheSizes(templateCaches)); + accumulateRecordCacheStats(templateCaches), + accumulateRecordCacheSizes(templateCaches), + accumulateRecordCacheMaxSizes(templateCaches), + avgTemplate); } @Nonnull @@ -336,9 +347,26 @@ }; } + @Nonnull + public static Supplier accumulateRecordCacheMaxSizes( + final Iterable> caches) { + return new Supplier() { + @Override + public Long get() { + long size = 0; + for (RecordCache cache : caches) { + size += cache.getMaxSize(); + } + return size; + } + }; + } + @CheckForNull @Override public CacheStatsMBean getNodeCacheStats() { + // TODO avg record entry + long avgRecord = 250; return new RecordCacheStats("Node deduplication cache stats", new Supplier() { @Override @@ -351,7 +379,8 @@ public Long get() { return nodeCache.get().size(); } - }); + }, + Suppliers.ofInstance(nodeCache.get().maxSize()), avgRecord); } @Override Index: src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java (working copy) @@ -79,6 +79,9 @@ "oak.segment.compaction.gcSizeDeltaEstimation", SIZE_DELTA_ESTIMATION_DEFAULT); + private double heapMultiplier = Double.parseDouble(System.getProperty( + "oak.segment.compaction.heapMultiplier", "1")); + public SegmentGCOptions(boolean paused, int gainThreshold, int retryCount, int forceTimeout) { this.paused = paused; this.gainThreshold = gainThreshold; @@ -294,4 +297,12 @@ return this; } + public double getHeapMultiplier() { + return heapMultiplier; + } + + public void setHeapMultiplier(double heapMultiplier) { + this.heapMultiplier = heapMultiplier; + } + } Index: src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (working copy) @@ -721,7 +721,24 @@ synchronized void run() throws IOException { gcListener.info("TarMK GC #{}: started", GC_COUNT.incrementAndGet()); - Stopwatch watch = Stopwatch.createStarted(); + + Runtime runtime = Runtime.getRuntime(); + long avail = runtime.maxMemory() - runtime.totalMemory() + + runtime.freeMemory(); + long required = (long) (estimateRequiredHeap() * gcOptions + .getHeapMultiplier()); + if (avail < required) { + gcListener + .info("TarMK GC #{}: compaction skipped because available memory {} ({} bytes) is too low, expecting at least {} ({} bytes)", + GC_COUNT, humanReadableByteCount(avail), avail, + humanReadableByteCount(required), required); + return; + } else { + gcListener + .info("TarMK GC #{}: available memory {} ({} bytes) is sufficient, expecting at least {} ({} bytes)", + GC_COUNT, humanReadableByteCount(avail), avail, + humanReadableByteCount(required), required); + } int gainThreshold = gcOptions.getGainThreshold(); boolean sufficientEstimatedGain = true; @@ -731,6 +748,7 @@ } else if (gcOptions.isPaused()) { gcListener.info("TarMK GC #{}: estimation skipped because compaction is paused", GC_COUNT); } else { + Stopwatch watch = Stopwatch.createStarted(); gcListener.info("TarMK GC #{}: estimation started", GC_COUNT); Supplier cancel = new CancelCompactionSupplier(FileStore.this); GCEstimation estimate = estimateCompactionGain(cancel); @@ -771,6 +789,26 @@ } } + private long estimateRequiredHeap() { + long est = delta(getNodeDeduplicationCacheStats()) + + delta(getStringDeduplicationCacheStats()) + + delta(getTemplateDeduplicationCacheStats()) + + delta(getSegmentCacheStats()) + + delta(getStringCacheStats()) + + delta(getTemplateCacheStats()); + return est; + } + + private long delta(CacheStatsMBean stats) { + if (stats != null && stats.getMaxTotalWeight() > 0) { + long max = stats.getMaxTotalWeight(); + long current = stats.estimateCurrentWeight(); + long delta = max - current; + return delta >= 0 ? delta : 0; + } + return 0; + } + /** * Estimated compaction gain. The result will be undefined if stopped through * the passed {@code stop} signal. Index: src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java (revision 1766632) +++ src/main/java/org/apache/jackrabbit/oak/segment/file/PriorityCache.java (working copy) @@ -137,6 +137,10 @@ return size; } + public long maxSize() { + return entries.length; + } + /** * Add a mapping to the cache. * @param key the key of the mapping