From d8a9a756df9c3e1414cfb554264122216fb6e73e Mon Sep 17 00:00:00 2001 From: Alex Parvulescu Date: Thu, 28 Jul 2016 16:23:59 +0200 Subject: [PATCH 1/2] persisted gc journal --- .../jackrabbit/oak/segment/file/FileStore.java | 3 +- .../oak/segment/file/FileStoreStats.java | 22 +++ .../oak/segment/file/GCJournalWriter.java | 195 +++++++++++++++++++++ .../oak/segment/file/FileStoreStatsTest.java | 24 +++ 4 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournalWriter.java diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java index 0284f45..7594765 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java @@ -291,7 +291,8 @@ public class FileStore implements SegmentStore, Closeable { long initialSize = size(); this.approximateSize = new AtomicLong(initialSize); - this.stats = new FileStoreStats(builder.getStatsProvider(), this, initialSize); + this.stats = new FileStoreStats(builder.getStatsProvider(), this, + initialSize).with(new GCJournalWriter(directory)); if (!readOnly) { if (indices.length > 0) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreStats.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreStats.java index 09cd3af..bfb7e21 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreStats.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreStats.java @@ -38,6 +38,7 @@ public class FileStoreStats implements FileStoreStatsMBean, FileStoreMonitor { private final FileStore store; private final MeterStats writeStats; private final CounterStats repoSize; + private GCJournalWriter gcJournal; public FileStoreStats(StatisticsProvider statisticsProvider, FileStore store, long initialSize) { this.statisticsProvider = statisticsProvider; @@ -47,6 +48,11 @@ public class FileStoreStats implements FileStoreStatsMBean, FileStoreMonitor { repoSize.inc(initialSize); } + public FileStoreStats with(GCJournalWriter gcJournal) { + this.gcJournal = gcJournal; + return this; + } + //~-----------------------------< FileStoreMonitor > @Override @@ -58,6 +64,22 @@ public class FileStoreStats implements FileStoreStatsMBean, FileStoreMonitor { @Override public void reclaimed(long size) { repoSize.dec(size); + persistGcJournal(); + } + + private void persistGcJournal() { + if (gcJournal != null) { + gcJournal.write(store.getHead().getRecordId().toString(), + getApproximateSize()); + } + } + + public long getPreviousCleanupSize() { + if (gcJournal != null) { + return gcJournal.read().getSize(); + } else { + return -1; + } } //~--------------------------------< FileStoreStatsMBean > diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournalWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournalWriter.java new file mode 100644 index 0000000..1f4c218 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournalWriter.java @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.file; + +import static com.google.common.base.Charsets.UTF_8; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.nio.file.Files.newBufferedWriter; +import static java.nio.file.Files.readAllLines; +import static java.nio.file.StandardOpenOption.APPEND; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.DSYNC; +import static java.nio.file.StandardOpenOption.WRITE; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Persists the repository meta following a cleanup operation in the + * {@link #GC_JOURNAL gc journal} file under the format: 'head record id, size, + * timestamp'. + */ +public class GCJournalWriter { + + private static final Logger LOG = LoggerFactory + .getLogger(GCJournalWriter.class); + + public static final String GC_JOURNAL = "gc.log"; + + @Nonnull + private final File directory; + + private GCJournalEntry latest; + + public GCJournalWriter(@Nonnull File directory) { + this.directory = checkNotNull(directory); + } + + public void write(String head, long size) { + latest = new GCJournalEntry(head, size, System.currentTimeMillis()); + Path path = new File(directory, GC_JOURNAL).toPath(); + try { + try (BufferedWriter w = newBufferedWriter(path, UTF_8, WRITE, + APPEND, CREATE, DSYNC)) { + w.write(latest.toString()); + w.newLine(); + } + } catch (IOException e) { + LOG.error("Error writing gc journal", e); + } + } + + private List readLines() { + File file = new File(directory, GC_JOURNAL); + if (file.exists()) { + try { + return readAllLines(file.toPath(), UTF_8); + } catch (IOException e) { + LOG.error("Error reading gc journal", e); + } + } + return new ArrayList(); + } + + public GCJournalEntry read() { + if (latest == null) { + List all = readLines(); + if (all.isEmpty()) { + latest = GCJournalEntry.EMPTY; + } else { + String info = all.get(all.size() - 1); + latest = GCJournalEntry.fromString(info); + } + } + return latest; + } + + public Collection readAll() { + List all = new ArrayList(); + for (String l : readLines()) { + all.add(GCJournalEntry.fromString(l)); + } + return all; + } + + static class GCJournalEntry { + + static GCJournalEntry EMPTY = new GCJournalEntry(null, -1, -1); + + private final String head; + private final long size; + private final long ts; + + public GCJournalEntry(String head, long size, long ts) { + this.head = head; + this.size = size; + this.ts = ts; + } + + @Override + public String toString() { + return head + "," + size + "," + ts; + } + + static GCJournalEntry fromString(String in) { + String[] items = in.split(","); + if (items.length == 3) { + String head = items[0]; + long ts = safeParse(items[1]); + long size = safeParse(items[2]); + return new GCJournalEntry(head, size, ts); + } + return GCJournalEntry.EMPTY; + } + + private static long safeParse(String in) { + try { + return Long.parseLong(in); + } catch (NumberFormatException ex) { + LOG.warn("Unable to parse {} as long value.", in, ex); + } + return -1; + } + + public String getHead() { + return head; + } + + public long getSize() { + return size; + } + + public long getTs() { + return ts; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((head == null) ? 0 : head.hashCode()); + result = prime * result + (int) (size ^ (size >>> 32)); + result = prime * result + (int) (ts ^ (ts >>> 32)); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + GCJournalEntry other = (GCJournalEntry) obj; + if (head == null) { + if (other.head != null) + return false; + } else if (!head.equals(other.head)) + return false; + if (size != other.size) + return false; + if (ts != other.ts) + return false; + return true; + } + + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java index f0b7dc1..fe178ff 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreStatsTest.java @@ -91,4 +91,28 @@ public class FileStoreStatsTest { } } + @Test + public void tarGcJournal() throws Exception { + File directory = segmentFolder.newFolder(); + StatisticsProvider statsProvider = new DefaultStatisticsProvider( + executor); + try (FileStore store = fileStoreBuilder(directory) + .withStatisticsProvider(statsProvider).build()) { + + GCJournalWriter gc = new GCJournalWriter(directory); + FileStoreStats stats = new FileStoreStats(statsProvider, store, + 1000).with(gc); + + long initial = stats.getApproximateSize(); + stats.reclaimed(150); + long r1 = initial - 150; + assertEquals(r1, stats.getPreviousCleanupSize()); + assertEquals(r1, gc.read().getSize()); + + stats.reclaimed(250); + long r2 = r1 - 250; + assertEquals(r2, stats.getPreviousCleanupSize()); + assertEquals(r2, gc.read().getSize()); + } + } } -- 1.8.4.3 From 260f486351a8205804068aa59d94e33118f9fa7b Mon Sep 17 00:00:00 2001 From: Alex Parvulescu Date: Mon, 8 Aug 2016 11:08:13 +0200 Subject: [PATCH 2/2] OAK-4293 Refactor / rework compaction gain estimation --- .../oak/segment/compaction/SegmentGCOptions.java | 16 ++++ .../oak/segment/compaction/SegmentRevisionGC.java | 4 + .../segment/compaction/SegmentRevisionGCMBean.java | 11 +++ .../oak/segment/file/CompactionGainEstimate.java | 58 ++++++++++++-- .../jackrabbit/oak/segment/file/FileStore.java | 40 ++++------ .../jackrabbit/oak/segment/file/GCEstimation.java | 35 ++++++++ .../oak/segment/file/SizeDeltaGcEstimation.java | 93 ++++++++++++++++++++++ .../oak/segment/file/CompactionEstimatorTest.java | 11 ++- 8 files changed, 236 insertions(+), 32 deletions(-) create mode 100644 oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java create mode 100644 oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java index 37337e2..3e337d6 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java @@ -84,6 +84,9 @@ public class SegmentGCOptions { "oak.segment.compaction.binaryDeduplicationMaxSize", 100 * 1024 * 1024); + private long gcSizeDeltaEstimation = Long.getLong( + "oak.segment.compaction.gcSizeDeltaEstimation", -1); + public SegmentGCOptions(boolean paused, int memoryThreshold, int gainThreshold, int retryCount, boolean forceAfterFail, int lockWaitTime) { this.paused = paused; @@ -326,4 +329,17 @@ public class SegmentGCOptions { public long getBinaryDeduplicationMaxSize() { return this.ocBinMaxSize; } + + public boolean isGcSizeDeltaEstimation() { + return gcSizeDeltaEstimation >= 0; + } + + public long getGcSizeDeltaEstimation() { + return gcSizeDeltaEstimation; + } + + public SegmentGCOptions setGcSizeDeltaEstimation(long gcSizeDeltaEstimation) { + this.gcSizeDeltaEstimation = gcSizeDeltaEstimation; + return this; + } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java index 4d6c817..1f64491 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java @@ -119,4 +119,8 @@ public interface SegmentRevisionGC { */ void setRetainedGenerations(int retainedGenerations); + long getGcSizeDeltaEstimation(); + + void setGcSizeDeltaEstimation(long gcSizeDeltaEstimation); + } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java index ffbf456..e17c0dc 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java @@ -102,4 +102,15 @@ public class SegmentRevisionGCMBean public void setRetainedGenerations(int retainedGenerations) { gcOptions.setRetainedGenerations(retainedGenerations); } + + @Override + public long getGcSizeDeltaEstimation() { + return gcOptions.getGcSizeDeltaEstimation(); + } + + @Override + public void setGcSizeDeltaEstimation(long gcSizeDeltaEstimation) { + gcOptions.setGcSizeDeltaEstimation(gcSizeDeltaEstimation); + } + } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionGainEstimate.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionGainEstimate.java index 5d3f6be..57286b8 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionGainEstimate.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionGainEstimate.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.segment.file; import static org.apache.jackrabbit.oak.api.Type.BINARIES; +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; import java.io.File; import java.util.UUID; @@ -36,7 +37,7 @@ import org.apache.jackrabbit.oak.segment.SegmentNodeState; import org.apache.jackrabbit.oak.segment.SegmentPropertyState; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; -class CompactionGainEstimate implements TarEntryVisitor { +class CompactionGainEstimate implements TarEntryVisitor, GCEstimation { private static final Funnel UUID_FUNNEL = new Funnel() { @Override @@ -48,10 +49,18 @@ class CompactionGainEstimate implements TarEntryVisitor { private final BloomFilter uuids; + private final int gainThreshold; + private long totalSize = 0; private long reachableSize = 0; + private boolean gcNeeded; + + private String gcInfo = "unknown"; + + private boolean finished = false; + /** * Create a new instance of gain estimator. The estimation process can be stopped * by switching the supplier {@code stop} to {@code true}, in which case the returned @@ -61,8 +70,10 @@ class CompactionGainEstimate implements TarEntryVisitor { * @param estimatedBulkCount * @param stop stop signal */ - CompactionGainEstimate(SegmentNodeState node, int estimatedBulkCount, Supplier stop) { + CompactionGainEstimate(SegmentNodeState node, int estimatedBulkCount, + Supplier stop, int gainThreshold) { uuids = BloomFilter.create(UUID_FUNNEL, estimatedBulkCount); + this.gainThreshold = gainThreshold; collectReferencedSegments(node, new RecordIdSet(), stop); } @@ -111,12 +122,47 @@ class CompactionGainEstimate implements TarEntryVisitor { return 100 * (totalSize - reachableSize) / totalSize; } - public long getTotalSize() { - return totalSize; + private void run() { + if (finished) { + return; + } + long gain = estimateCompactionGain(); + gcNeeded = gain >= gainThreshold; + if (gcNeeded) { + gcInfo = String + .format("Gain is %s%% or %s/%s (%s/%s bytes), so running compaction", + gain, humanReadableByteCount(reachableSize), + humanReadableByteCount(totalSize), + reachableSize, totalSize); + } else { + if (totalSize == 0) { + gcInfo = "Skipping compaction for now as repository consists of a single tar file only"; + } else { + gcInfo = String + .format("Gain is %s%% or %s/%s (%s/%s bytes), so skipping compaction for now", + gain, + humanReadableByteCount(reachableSize), + humanReadableByteCount(totalSize), + reachableSize, totalSize); + } + } + finished = true; } - public long getReachableSize() { - return reachableSize; + @Override + public boolean gcNeeded() { + if (!finished) { + run(); + } + return gcNeeded; + } + + @Override + public String gcLog() { + if (!finished) { + run(); + } + return gcInfo; } // ---------------------------------------------------< TarEntryVisitor >-- diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java index acb81b6..eb002c4 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java @@ -468,34 +468,21 @@ public class FileStore implements SegmentStore, Closeable { } else { gcListener.info("TarMK GC #{}: estimation started", GC_COUNT); Supplier shutdown = newShutdownSignal(); - CompactionGainEstimate estimate = estimateCompactionGain(shutdown); + GCEstimation estimate = estimateCompactionGain(shutdown); if (shutdown.get()) { gcListener.info("TarMK GC #{}: estimation interrupted. Skipping compaction.", GC_COUNT); } - long gain = estimate.estimateCompactionGain(); - sufficientEstimatedGain = gain >= gainThreshold; + sufficientEstimatedGain = estimate.gcNeeded(); + String gcLog = estimate.gcLog(); if (sufficientEstimatedGain) { gcListener.info( - "TarMK GC #{}: estimation completed in {} ({} ms). " + - "Gain is {}% or {}/{} ({}/{} bytes), so running compaction", - GC_COUNT, watch, watch.elapsed(MILLISECONDS), gain, - humanReadableByteCount(estimate.getReachableSize()), humanReadableByteCount(estimate.getTotalSize()), - estimate.getReachableSize(), estimate.getTotalSize()); + "TarMK GC #{}: estimation completed in {} ({} ms). {}", + GC_COUNT, watch, watch.elapsed(MILLISECONDS), gcLog); } else { - if (estimate.getTotalSize() == 0) { - gcListener.skipped( - "TarMK GC #{}: estimation completed in {} ({} ms). " + - "Skipping compaction for now as repository consists of a single tar file only", - GC_COUNT, watch, watch.elapsed(MILLISECONDS)); - } else { - gcListener.skipped( - "TarMK GC #{}: estimation completed in {} ({} ms). " + - "Gain is {}% or {}/{} ({}/{} bytes), so skipping compaction for now", - GC_COUNT, watch, watch.elapsed(MILLISECONDS), gain, - humanReadableByteCount(estimate.getReachableSize()), humanReadableByteCount(estimate.getTotalSize()), - estimate.getReachableSize(), estimate.getTotalSize()); - } + gcListener.skipped( + "TarMK GC #{}: estimation completed in {} ({} ms). {}", + GC_COUNT, watch, watch.elapsed(MILLISECONDS), gcLog); } } @@ -657,8 +644,15 @@ public class FileStore implements SegmentStore, Closeable { * @param stop signal for stopping the estimation process. * @return compaction gain estimate */ - CompactionGainEstimate estimateCompactionGain(Supplier stop) { - CompactionGainEstimate estimate = new CompactionGainEstimate(getHead(), count(), stop); + GCEstimation estimateCompactionGain(Supplier stop) { + if (gcOptions.isGcSizeDeltaEstimation()) { + SizeDeltaGcEstimation e = new SizeDeltaGcEstimation(gcOptions, + stats); + return e; + } + + CompactionGainEstimate estimate = new CompactionGainEstimate(getHead(), + count(), stop, gcOptions.getGainThreshold()); fileStoreLock.readLock().lock(); try { for (TarReader reader : readers) { diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java new file mode 100644 index 0000000..70bbca5 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCEstimation.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +public interface GCEstimation { + + /** + * Determines if the Gc operation needs to run or not + */ + boolean gcNeeded(); + + /** + * User friendly message explaining the value of the + * {@link GCEstimation#isGcNeeded()} flag + */ + String gcLog(); + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java new file mode 100644 index 0000000..9c151d5 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGcEstimation.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file; + +import static java.lang.String.format; +import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; + +import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; + +public class SizeDeltaGcEstimation implements GCEstimation { + + private final long delta; + + private final FileStoreStats stats; + + private boolean gcNeeded; + + private String gcInfo = "unknown"; + + private boolean finished = false; + + public SizeDeltaGcEstimation(SegmentGCOptions opts, FileStoreStats stats) { + this.delta = opts.getGcSizeDeltaEstimation(); + this.stats = stats; + } + + @Override + public boolean gcNeeded() { + if (!finished) { + run(); + } + return gcNeeded; + } + + @Override + public String gcLog() { + if (!finished) { + run(); + } + return gcInfo; + } + + private void run() { + if (finished) { + return; + } + if (delta == 0) { + gcNeeded = true; + gcInfo = format( + "Estimation skipped because of size delta value (%s <= 0)", + delta); + } else if (stats.getPreviousCleanupSize() < 0) { + gcNeeded = true; + gcInfo = format("Estimation skipped because of missing gc journal data"); + } else { + long totalSize = stats.getApproximateSize(); + long lastGc = stats.getPreviousCleanupSize(); + long gain = totalSize - lastGc; + long gainP = 100 * (totalSize - lastGc) / totalSize; + + gcNeeded = gain > delta; + if (gcNeeded) { + gcInfo = format( + "Size Delta is %s%% or %s/%s (%s/%s bytes), so running compaction", + gainP, humanReadableByteCount(lastGc), + humanReadableByteCount(totalSize), lastGc, totalSize); + } else { + gcInfo = format( + "Size Delta is %s%% or %s/%s (%s/%s bytes), so skipping compaction for now", + gainP, humanReadableByteCount(lastGc), + humanReadableByteCount(totalSize), lastGc, totalSize); + } + } + finished = true; + } +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionEstimatorTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionEstimatorTest.java index fdf7a44..cebbf49 100644 --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionEstimatorTest.java +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/CompactionEstimatorTest.java @@ -80,9 +80,14 @@ public class CompactionEstimatorTest { fileStore.flush(); try { - // should be at 66% - assertTrue(fileStore.estimateCompactionGain(Suppliers.ofInstance(false)) - .estimateCompactionGain() > 60); + GCEstimation est = fileStore.estimateCompactionGain(Suppliers + .ofInstance(false)); + assertTrue(est.gcNeeded()); + if (est instanceof CompactionGainEstimate) { + // should be at 66% + assertTrue(((CompactionGainEstimate) est) + .estimateCompactionGain() > 60); + } } finally { fileStore.close(); } -- 1.8.4.3