Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Checkpoints.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Checkpoints.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/Checkpoints.java (working copy) @@ -31,6 +31,7 @@ import org.apache.jackrabbit.oak.commons.json.JsopTokenizer; import org.apache.jackrabbit.oak.commons.json.JsopWriter; import org.apache.jackrabbit.oak.plugins.document.util.Utils; +import org.apache.jackrabbit.oak.stats.Clock; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; @@ -89,17 +90,22 @@ rv[0] = nodeStore.getHeadRevision(); } }); - createCounter.getAndIncrement(); - performCleanupIfRequired(); - UpdateOp op = new UpdateOp(ID, false); - long endTime = BigInteger.valueOf(nodeStore.getClock().getTime()) - .add(BigInteger.valueOf(lifetimeInMillis)) - .min(BigInteger.valueOf(Long.MAX_VALUE)).longValue(); - op.setMapEntry(PROP_CHECKPOINT, r, new Info(endTime, rv[0], info).toString()); - store.createOrUpdate(Collection.SETTINGS, op); + long endTime = calculateEndTime(nodeStore.getClock(), lifetimeInMillis); + create(r, new Info(endTime, rv[0], info)); return r; } + public Revision create(long lifetimeInMillis, + @NotNull Map info, + @NotNull Revision revision) { + if (revision.getTimestamp() > nodeStore.getClock().getTime()) { + throw new IllegalArgumentException("Cannot create checkpoint with a revision in the future: " + revision); + } + long endTime = calculateEndTime(nodeStore.getClock(), lifetimeInMillis); + create(revision, new Info(endTime, null, info)); + return revision; + } + public void release(String checkpoint) { UpdateOp op = new UpdateOp(ID, false); op.removeMapEntry(PROP_CHECKPOINT, Revision.fromString(checkpoint)); @@ -239,6 +245,21 @@ } } + private void create(@NotNull Revision revision, @NotNull Info info) { + createCounter.getAndIncrement(); + performCleanupIfRequired(); + UpdateOp op = new UpdateOp(ID, false); + op.setMapEntry(PROP_CHECKPOINT, revision, info.toString()); + store.createOrUpdate(Collection.SETTINGS, op); + } + + private static long calculateEndTime(@NotNull Clock clock, + long lifetimeMs) { + return BigInteger.valueOf(clock.getTime()) + .add(BigInteger.valueOf(lifetimeMs)) + .min(BigInteger.valueOf(Long.MAX_VALUE)).longValue(); + } + private RevisionVector expand(Revision checkpoint) { LOG.warn("Expanding {} single revision checkpoint into a " + "RevisionVector. Please make sure all cluster nodes run " + Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (working copy) @@ -3468,7 +3468,8 @@ try { return new DocumentNodeStoreMBeanImpl(this, builder.getStatisticsProvider().getStats(), - clusterNodes.values()); + clusterNodes.values(), + builder.getRevisionGCMaxAge()); } catch (NotCompliantMBeanException e) { throw new IllegalStateException(e); } Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreBuilder.java (working copy) @@ -21,6 +21,7 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; import com.google.common.base.Predicate; import com.google.common.base.Predicates; @@ -65,6 +66,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Suppliers.ofInstance; import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS; +import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_VER_GC_MAX_AGE; /** * A generic builder for a {@link DocumentNodeStore}. By default the builder @@ -148,6 +150,7 @@ private int commitValueCacheSize = 10000; private boolean cacheEmptyCommitValue = false; private long maxRevisionAgeMillis = DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS; + private long maxRevisionGCAgeMillis = TimeUnit.SECONDS.toMillis(DEFAULT_VER_GC_MAX_AGE); private GCMonitor gcMonitor = new LoggingGCMonitor( LoggerFactory.getLogger(VersionGarbageCollector.class)); private Predicate nodeCachePredicate = Predicates.alwaysTrue(); @@ -611,6 +614,21 @@ return maxRevisionAgeMillis; } + public T setRevisionGCMaxAge(long maxRevisionGCAgeMillis) { + this.maxRevisionGCAgeMillis = maxRevisionGCAgeMillis; + return thisBuilder(); + } + + /** + * The maximum age for changes in milliseconds. Older changes are candidates + * for revision garbage collection. + * + * @return maximum age in milliseconds. + */ + public long getRevisionGCMaxAge() { + return maxRevisionGCAgeMillis; + } + public T setGCMonitor(@NotNull GCMonitor gcMonitor) { this.gcMonitor = checkNotNull(gcMonitor); return thisBuilder(); Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java (working copy) @@ -86,4 +86,18 @@ @Description("Possible values are: DIFF, NODE, NODECHILDREN") @Name("name") String name); + + @Description("Creates a checkpoint with the given revision and lifetime.") + String createCheckpoint( + @Description("The revision of the checkpoint to create.") + @Name("revision") + String revision, + @Description("The lifetime of the checkpoint in milliseconds.") + @Name("lifetime") + long lifetime, + @Description("Force create the checkpoint even when the revision " + + "garbage collector may have cleaned up changes already " + + "that are more recent than the revision checkpoint.") + @Name("force") + boolean force); } Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBeanImpl.java (working copy) @@ -40,6 +40,7 @@ import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.toArray; import static com.google.common.collect.Iterables.transform; +import static java.util.Collections.emptyMap; /** * Implementation of a DocumentNodeStoreMBean. @@ -52,16 +53,19 @@ private final DocumentNodeStore nodeStore; private final RepositoryStatistics repoStats; private final Iterable clusterNodes; + private final long revisionGCMaxAgeMillis; private final Logger log = LoggerFactory.getLogger(this.getClass()); DocumentNodeStoreMBeanImpl(DocumentNodeStore nodeStore, RepositoryStatistics repoStats, - Iterable clusterNodes) + Iterable clusterNodes, + long revisionGCMaxAgeMillis) throws NotCompliantMBeanException { super(DocumentNodeStoreMBean.class); this.nodeStore = nodeStore; this.repoStats = repoStats; this.clusterNodes = clusterNodes; + this.revisionGCMaxAgeMillis = revisionGCMaxAgeMillis; } @Override @@ -264,4 +268,24 @@ return "ERROR: Invalid cache name received."; } } + + @Override + public String createCheckpoint(String revision, long lifetime, boolean force) { + Revision rev = Revision.fromString(revision); + long oldestTimestamp = nodeStore.getClock().getTime() - revisionGCMaxAgeMillis; + Revision oldestCheckpoint = nodeStore.getCheckpoints().getOldestRevisionToKeep(); + if (oldestCheckpoint != null) { + oldestTimestamp = Math.min(oldestTimestamp, oldestCheckpoint.getTimestamp()); + } + if (force || rev.getTimestamp() < oldestTimestamp) { + String cp = nodeStore.getCheckpoints().create(lifetime, emptyMap(), rev).toString(); + log.info("Created checkpoint [{}] with lifetime {} for Revision {}", cp, lifetime, revision); + return cp; + } else { + throw new IllegalArgumentException(String.format("Cannot create a checkpoint for revision %s. " + + "Revision timestamp is %d and oldest timestamp to keep is %d", + revision, rev.getTimestamp(), oldestTimestamp)); + } + } + } Index: oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java =================================================================== --- oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (revision 1886301) +++ oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (working copy) @@ -325,6 +325,7 @@ } mkBuilder.setGCMonitor(new DelegatingGCMonitor( newArrayList(gcMonitor, loggingGCMonitor))); + mkBuilder.setRevisionGCMaxAge(TimeUnit.SECONDS.toMillis(config.versionGcMaxAgeInSecs())); nodeStore = mkBuilder.build(); Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsTest.java (revision 1886301) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/CheckpointsTest.java (working copy) @@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.stats.Clock; +import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -34,16 +35,19 @@ import com.google.common.collect.ImmutableMap; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNotSame; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; public class CheckpointsTest { + private static final long ONE_HOUR = TimeUnit.HOURS.toMillis(1); + @Rule public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); @@ -51,10 +55,14 @@ private DocumentNodeStore store; + private Checkpoints checkpoints; + @Before public void setUp() throws InterruptedException { clock = new Clock.Virtual(); + clock.waitUntil(System.currentTimeMillis()); store = builderProvider.newBuilder().clock(clock).getNodeStore(); + checkpoints = new Checkpoints(store); } @Test @@ -418,4 +426,31 @@ assertTrue(root.hasChildNode("foo")); assertFalse(root.hasChildNode("bar")); } + + @Test + public void createCheckpointWithRevision() { + Revision r = new Revision(clock.getTime(), 0, store.getClusterId()); + assertNotNull(r); + assertEquals(r, checkpoints.create(ONE_HOUR, Collections.emptyMap(), r)); + RevisionVector rv = checkpoints.retrieve(r.toString()); + assertNotNull(rv); + } + + @Test + public void createCheckpointWithRevisionTwice() { + Revision r = new Revision(clock.getTime(), 0, store.getClusterId()); + assertNotNull(r); + assertEquals(r, checkpoints.create(ONE_HOUR, Collections.emptyMap(), r)); + assertEquals(r, checkpoints.create(ONE_HOUR * 3, Collections.emptyMap(), r)); + Checkpoints.Info info = checkpoints.getCheckpoints().get(r); + assertNotNull(info); + assertThat(info.getExpiryTime(), greaterThan(store.getClock().getTime() + ONE_HOUR * 2)); + } + + @Test + public void createCheckpointWithRevisionInFuture() { + long time = store.getClock().getTime() + ONE_HOUR; + Revision r = new Revision(time, 0, store.getClusterId()); + Assert.assertThrows(IllegalArgumentException.class, () -> checkpoints.create(ONE_HOUR, Collections.emptyMap(), r)); + } } Index: oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java =================================================================== --- oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java (revision 1886303) +++ oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreTest.java (working copy) @@ -135,6 +135,8 @@ private static final Logger LOG = LoggerFactory.getLogger(DocumentNodeStoreTest.class); + private static final long ONE_MINUTE = TimeUnit.MINUTES.toMillis(1); + @Rule public DocumentMKBuilderProvider builderProvider = new DocumentMKBuilderProvider(); @@ -4092,6 +4094,52 @@ } // End of tests for OAK-9300 + @Test + public void createCheckpointWithRevision() throws Exception { + DocumentNodeStore ns = builderProvider.newBuilder().getNodeStore(); + RevisionVector head = ns.getHeadRevision(); + NodeBuilder builder = ns.getRoot().builder(); + builder.child("foo"); + merge(ns, builder); + Revision r = head.getRevision(ns.getClusterId()); + assertNotNull(r); + String ref = ns.getCheckpoints().create(ONE_MINUTE, Collections.emptyMap(), r).toString(); + NodeState root = ns.retrieve(ref); + assertNotNull(root); + assertFalse(root.hasChildNode("foo")); + } + + @Test + public void expandCheckpointWithRevision() throws Exception { + DocumentStore store = new MemoryDocumentStore(); + DocumentNodeStore ns1 = builderProvider.newBuilder() + .setDocumentStore(store).setAsyncDelay(0).setClusterId(1) + .getNodeStore(); + NodeBuilder builder = ns1.getRoot().builder(); + builder.child("foo"); + merge(ns1, builder); + ns1.runBackgroundOperations(); + RevisionVector head = ns1.getHeadRevision(); + + DocumentNodeStore ns2 = builderProvider.newBuilder() + .setDocumentStore(store).setAsyncDelay(0).setClusterId(2) + .getNodeStore(); + builder = ns2.getRoot().builder(); + builder.child("bar"); + merge(ns2, builder); + ns2.runBackgroundOperations(); + ns1.runBackgroundOperations(); + assertTrue(ns1.getRoot().hasChildNode("bar")); + + Revision r = head.getRevision(ns1.getClusterId()); + assertNotNull(r); + String ref = ns1.getCheckpoints().create(ONE_MINUTE, Collections.emptyMap(), r).toString(); + NodeState root = ns1.retrieve(ref); + assertNotNull(root); + assertTrue(root.hasChildNode("foo")); + assertFalse(root.hasChildNode("bar")); + } + private void getChildNodeCountTest(int numChildren, Iterable maxValues, Iterable expectedValues)