diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java index ebfc6eba82..9a1185a2df 100755 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java @@ -16,6 +16,8 @@ */ package org.apache.jackrabbit.oak.upgrade; +import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Calendar; import java.util.LinkedHashMap; @@ -23,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import javax.jcr.RepositoryException; @@ -31,6 +34,13 @@ import org.apache.commons.lang.StringUtils; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeBuilder; +import org.apache.jackrabbit.oak.segment.RecordId; +import org.apache.jackrabbit.oak.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.segment.SegmentReader; +import org.apache.jackrabbit.oak.segment.SegmentWriter; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.commit.CommitHook; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.CompositeEditorProvider; @@ -40,9 +50,11 @@ import org.apache.jackrabbit.oak.spi.state.ApplyDiff; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.LoggingCompositeHook; import org.apache.jackrabbit.oak.upgrade.checkpoint.CheckpointRetriever; +import org.apache.jackrabbit.oak.upgrade.cli.node.SegmentTarFactory; import org.apache.jackrabbit.oak.upgrade.cli.node.TarNodeStore; import org.apache.jackrabbit.oak.upgrade.nodestate.FilteringNodeState; import org.apache.jackrabbit.oak.upgrade.nodestate.MetadataExposingNodeState; @@ -57,11 +69,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableSet.copyOf; import static com.google.common.collect.ImmutableSet.of; import static com.google.common.collect.Sets.union; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import static org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.NT_REP_PERMISSION_STORE; import static org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionConstants.REP_PERMISSION_STORE; import static org.apache.jackrabbit.oak.upgrade.RepositoryUpgrade.DEFAULT_EXCLUDE_FRAGMENTS; @@ -89,6 +103,8 @@ public class RepositorySidegrade { */ private final NodeStore target; + private final FileStore targetFileStore; + private final NodeStore source; /** @@ -180,6 +196,16 @@ public class RepositorySidegrade { public RepositorySidegrade(NodeStore source, NodeStore target) { this.source = source; this.target = target; + + FileStore fs = null; + if (target instanceof TarNodeStore) { + TarNodeStore tarNodeStore = (TarNodeStore) target; + TarNodeStore.SuperRootProvider superRootProvider = tarNodeStore.getSuperRootProvider(); + if (superRootProvider instanceof SegmentTarFactory.SegmentTarSuperRootProvider) { + fs = ((SegmentTarFactory.SegmentTarSuperRootProvider) superRootProvider).getFileStore(); + } + } + this.targetFileStore = fs; } /** @@ -315,7 +341,7 @@ public class RepositorySidegrade { builder.setChildNode(":async"); } - private void copyState() throws CommitFailedException, RepositoryException { + private void copyState() throws CommitFailedException, RepositoryException, IOException { boolean migrateCheckpoints = true; if (!isCompleteMigration() && !forceCheckpoints) { LOG.info("Checkpoints won't be migrated because of the specified paths"); @@ -341,7 +367,7 @@ public class RepositorySidegrade { } } - private boolean migrateWithCheckpoints() throws CommitFailedException { + private boolean migrateWithCheckpoints() throws CommitFailedException, IOException { List checkpoints = CheckpointRetriever.getCheckpoints(source); if (checkpoints == null) { return false; @@ -352,7 +378,6 @@ public class RepositorySidegrade { NodeState initialRoot = target.getRoot(); NodeState previousRoot = initialRoot; - NodeBuilder targetRoot = previousRoot.builder(); for (CheckpointRetriever.Checkpoint checkpoint : checkpoints) { NodeState checkpointRoot = source.retrieve(checkpoint.getName()); Map checkpointInfo = source.checkpointInfo(checkpoint.getName()); @@ -367,11 +392,7 @@ public class RepositorySidegrade { } LOG.info("Checkpoint expiry time: {}, metadata: {}", checkpoint.getExpiryTime(), checkpointInfo); - NodeState currentRoot = wrapNodeState(checkpointRoot, tracePaths, true); - NodeState baseRoot = wrapNodeState(previousRoot, false, true); - currentRoot.compareAgainstBaseState(baseRoot, new ApplyDiff(targetRoot)); - - target.merge(targetRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY); + copyDiffToTarget(previousRoot, checkpointRoot, tracePaths); previousRoot = checkpointRoot; String newCheckpointName = target.checkpoint(checkpoint.getExpiryTime() - System.currentTimeMillis(), checkpointInfo); @@ -390,12 +411,12 @@ public class RepositorySidegrade { LOG.info("Applying diff to head"); tracePaths = false; } - - NodeState currentRoot = wrapNodeState(sourceRoot, tracePaths, true); - NodeState baseRoot = wrapNodeState(previousRoot, false, true); - currentRoot.compareAgainstBaseState(baseRoot, new ApplyDiff(targetRoot)); + + copyDiffToTarget(previousRoot, sourceRoot, tracePaths); LOG.info("Rewriting checkpoint names in /:async {}", nameToRevision); + + NodeBuilder targetRoot = target.getRoot().builder(); NodeBuilder async = targetRoot.getChildNode(":async"); for (Map.Entry e : nameToRevision.entrySet()) { async.setProperty(e.getKey(), e.getValue(), Type.STRING); @@ -412,11 +433,28 @@ public class RepositorySidegrade { } async.setProperty(e.getKey() + "-temp", tempValues, Type.STRINGS); } - target.merge(targetRoot, EmptyHook.INSTANCE, CommitInfo.EMPTY); return true; } + private void copyDiffToTarget(NodeState before, NodeState after, boolean tracePaths) throws IOException, CommitFailedException { + NodeState currentRoot = wrapNodeState(after, tracePaths, true); + NodeState baseRoot = wrapNodeState(before, false, true); + + NodeBuilder targetBuilder; + if (targetFileStore == null) { + targetBuilder = target.getRoot().builder(); + currentRoot.compareAgainstBaseState(baseRoot, new ApplyDiff(targetBuilder)); + } else { + SegmentNodeState state = new PersistingDiff(target.getRoot()).diff(before, after); + + NodeState targetRoot = target.getRoot(); + targetBuilder = targetRoot.builder(); + state.compareAgainstBaseState(targetRoot, new ApplyDiff(targetBuilder)); + } + target.merge(targetBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } + private void migrateWithoutCheckpoints() throws CommitFailedException, RepositoryException { final List hooks = new ArrayList<>(); if (customCommitHooks != null) { @@ -553,4 +591,147 @@ public class RepositorySidegrade { private boolean targetExists() { return target.getRoot().getChildNodeEntries().iterator().hasNext(); } + + private class PersistingDiff implements NodeStateDiff { + + /** + * Number of content updates that need to happen before the updates + * are automatically purged to the underlying segments. + */ + final int UPDATE_LIMIT = + Integer.getInteger("upgrade.update.limit", 10000); + + private final SegmentWriter writer; + + private final SegmentReader reader; + + private final BlobStore blobStore; + + @Nonnull + private MemoryNodeBuilder builder; + + @Nonnull + private final NodeState base; + + @CheckForNull + private IOException exception; + + private long modCount; + + PersistingDiff(@Nonnull NodeState base) { + this.writer = targetFileStore.getWriter(); + this.reader = targetFileStore.getReader(); + this.blobStore = targetFileStore.getBlobStore(); + this.builder = new MemoryNodeBuilder(checkNotNull(base)); + this.base = base; + } + + private SegmentNodeState compare(NodeState state) throws IOException { + return compare(EMPTY_NODE, state, EMPTY_NODE); + } + + public SegmentNodeState compare( + @Nonnull NodeState before, + @Nonnull NodeState after, + @Nonnull NodeState onto) throws IOException { + return new PersistingDiff(onto).diff(before, after); + } + + private void updated() throws IOException { + if (++modCount % UPDATE_LIMIT == 0) { + RecordId newBaseId = writer.writeNode(builder.getNodeState(), null); + SegmentNodeState newBase = new SegmentNodeState(reader, writer, blobStore, newBaseId); + builder = new MemoryNodeBuilder(newBase); + } + } + + @CheckForNull + SegmentNodeState diff(@Nonnull NodeState before, @Nonnull NodeState after) throws IOException { + boolean success = after.compareAgainstBaseState(before, this); + if (exception != null) { + throw new IOException(exception); + } else if (success) { + NodeState nodeState = builder.getNodeState(); + checkState(modCount == 0 || !(nodeState instanceof SegmentNodeState)); + RecordId nodeId = writer.writeNode(nodeState, getStableIdBytes(after)); + return new SegmentNodeState(reader, writer, blobStore, nodeId); + } else { + return null; + } + } + + @Override + public boolean propertyAdded(@Nonnull PropertyState after) { + builder.setProperty(after); + return true; + } + + @Override + public boolean propertyChanged(@Nonnull PropertyState before, @Nonnull PropertyState after) { + builder.setProperty(after); + return true; + } + + @Override + public boolean propertyDeleted(PropertyState before) { + builder.removeProperty(before.getName()); + return true; + } + + @Override + public boolean childNodeAdded(@Nonnull String name, @Nonnull NodeState after) { + try { + SegmentNodeState compacted = compare(after); + if (compacted != null) { + updated(); + builder.setChildNode(name, compacted); + return true; + } else { + return false; + } + } catch (IOException e) { + exception = e; + return false; + } + } + + @Override + public boolean childNodeChanged(@Nonnull String name, @Nonnull NodeState before, @Nonnull NodeState after) { + try { + SegmentNodeState compacted = compare(before, after, base.getChildNode(name)); + if (compacted != null) { + updated(); + builder.setChildNode(name, compacted); + return true; + } else { + return false; + } + } catch (IOException e) { + exception = e; + return false; + } + } + + @Override + public boolean childNodeDeleted(String name, NodeState before) { + try { + updated(); + builder.getChildNode(name).remove(); + return true; + } catch (IOException e) { + exception = e; + return false; + } + } + + @CheckForNull + private ByteBuffer getStableIdBytes(NodeState state) { + if (state instanceof SegmentNodeState) { + return ((SegmentNodeState) state).getStableIdBytes(); + } else { + return null; + } + } + } + } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java index 9bc2c895c6..7db30dbb5d 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/SegmentTarFactory.java @@ -154,7 +154,7 @@ public class SegmentTarFactory implements NodeStoreFactory { private static class ExternalBlobFound extends RuntimeException { } - private static class SegmentTarSuperRootProvider implements TarNodeStore.SuperRootProvider { + public static class SegmentTarSuperRootProvider implements TarNodeStore.SuperRootProvider { private final ReadOnlyFileStore readOnlyFileStore; @@ -170,6 +170,10 @@ public class SegmentTarFactory implements NodeStoreFactory { this.fileStore = fileStore; } + public FileStore getFileStore() { + return fileStore; + } + @Override public NodeState getSuperRoot() { return fileStore == null ? readOnlyFileStore.getHead() : fileStore.getHead(); diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java index 0e43936471..89b6f73794 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/node/TarNodeStore.java @@ -40,7 +40,11 @@ public class TarNodeStore extends ProxyNodeStore { return ns; } - interface SuperRootProvider { + public SuperRootProvider getSuperRootProvider() { + return superRootProvider; + } + + public interface SuperRootProvider { NodeState getSuperRoot();