diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java new file mode 100644 index 0000000..7337997 --- /dev/null +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/IndexCopier.java @@ -0,0 +1,134 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.upgrade; + +import org.apache.commons.lang.StringUtils; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.upgrade.nodestate.NodeStateCopier; + +import java.util.Set; + +import static java.util.Collections.singleton; +import static org.apache.commons.lang.StringUtils.isEmpty; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_CONTENT_NODE_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.UNIQUE_PROPERTY_NAME; + +public class IndexCopier { + + /** + * Copy all index definition and data from sourceRoot to targetRoot. The + * indexing data is filtered to include only content related to the passed + * list of paths. + * + * @param sourceRoot the source + * @param targetRoot the target + * @param includes indexing data for these paths will be copied + */ + public static void copy(NodeState sourceRoot, NodeBuilder targetRoot, Set includes) { + NodeState oakIndex = sourceRoot.getChildNode(INDEX_DEFINITIONS_NAME); + NodeBuilder targetOakIndex = copySingleNode(oakIndex, targetRoot, INDEX_DEFINITIONS_NAME); + + for (ChildNodeEntry child : oakIndex.getChildNodeEntries()) { + NodeState indexDef = child.getNodeState(); + String type = indexDef.getString(TYPE_PROPERTY_NAME); + if (isEmpty(type)) { + continue; + } + + NodeBuilder targetIndexDef = copySingleNode(child, targetOakIndex); + + switch (type) { + case "property": + if (indexDef.getBoolean(UNIQUE_PROPERTY_NAME)) { + copyUniqueIndex(indexDef, targetIndexDef, includes); + } else { + copyMirrorIndex(indexDef, targetIndexDef, includes); + } + break; + + case "counter": + copyMirrorIndex(indexDef, targetIndexDef, includes); + + case "lucene": + copyLuceneIndex(indexDef, targetIndexDef, includes); + + default: + break; + } + } + } + + private static void copyMirrorIndex(NodeState indexDef, NodeBuilder targetIndexDef, Set includes) { + NodeState indexNode = indexDef.getChildNode(INDEX_CONTENT_NODE_NAME); + NodeBuilder targetIndexNode = copySingleNode(indexNode, targetIndexDef, INDEX_CONTENT_NODE_NAME); + + for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) { + NodeBuilder targetAttr = copySingleNode(attr, targetIndexNode); + NodeStateCopier.builder() + .include(includes) + .copy(attr.getNodeState(), targetAttr); + } + } + + private static void copyUniqueIndex(NodeState indexDef, NodeBuilder targetIndexDef, Set includes) { + NodeState indexNode = indexDef.getChildNode(INDEX_CONTENT_NODE_NAME); + NodeBuilder targetIndexNode = copySingleNode(indexNode, targetIndexDef, INDEX_CONTENT_NODE_NAME); + + for (ChildNodeEntry attr : indexNode.getChildNodeEntries()) { + Iterable entries = attr.getNodeState().getStrings("entry"); + if (entries != null) { + for (String e : entries) { + if (startsWithAny(e, includes)) { + copySingleNode(attr, targetIndexNode); + } + } + } + } + } + + private static void copyLuceneIndex(NodeState indexDef, NodeBuilder targetIndexDef, Set includes) { + NodeStateCopier.builder() + .include(singleton("/")) + .copy(indexDef, targetIndexDef); + } + + private static NodeBuilder copySingleNode(ChildNodeEntry source, NodeBuilder targetParent) { + return copySingleNode(source.getNodeState(), targetParent, source.getName()); + } + + private static NodeBuilder copySingleNode(NodeState source, NodeBuilder targetParent, String name) { + NodeBuilder target = targetParent.child(name); + for (PropertyState p : source.getProperties()) { + target.setProperty(p); + } + return target; + } + + private static boolean startsWithAny(String subject, Iterable patterns) { + for (String p : patterns) { + if (subject.startsWith(p)) { + return true; + } + } + return false; + } +} diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java index b603782..a9c9710 100755 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/RepositorySidegrade.java @@ -91,6 +91,8 @@ public class RepositorySidegrade { */ private Set mergePaths = DEFAULT_MERGE_PATHS; + private boolean includeIndex = false; + private boolean filterLongNames = true; private boolean skipInitialization = false; @@ -187,6 +189,9 @@ public class RepositorySidegrade { this.excludePaths = copyOf(checkNotNull(excludes)); } + public void setIncludeIndex(boolean includeIndex) { + this.includeIndex = includeIndex; + } /** * Sets the paths that should be merged when the source repository @@ -273,8 +278,19 @@ public class RepositorySidegrade { private void copyState(NodeState sourceRoot, NodeBuilder targetRoot) throws CommitFailedException { copyWorkspace(sourceRoot, targetRoot); + if (includeIndex) { + IndexCopier.copy(sourceRoot, targetRoot, includePaths); + } + + boolean isRemoveCheckpointReferences = false; if (!copyCheckpoints(targetRoot)) { LOG.info("Copying checkpoints is not supported for this combination of node stores"); + isRemoveCheckpointReferences = true; + } + if (!DEFAULT_INCLUDE_PATHS.equals(includePaths)) { + isRemoveCheckpointReferences = true; + } + if (isRemoveCheckpointReferences) { removeCheckpointReferences(targetRoot); } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java index 159e5c1..360de24 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/MigrationFactory.java @@ -117,6 +117,7 @@ public class MigrationFactory { } sidegrade.setFilterLongNames(stores.getSrcType().isSupportLongNames() && !stores.getDstType().isSupportLongNames()); sidegrade.setSkipInitialization(options.isSkipInitialization()); + sidegrade.setIncludeIndex(options.isIncludeIndex()); return sidegrade; } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java index be20b83..c837f2c 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/MigrationOptions.java @@ -46,6 +46,8 @@ public class MigrationOptions { private final String[] mergePaths; + private final boolean includeIndex; + private final boolean failOnError; private final boolean earlyShutdown; @@ -78,6 +80,7 @@ public class MigrationOptions { this.includePaths = split(args.getOption(OptionParserFactory.INCLUDE_PATHS)); this.excludePaths = split(args.getOption(OptionParserFactory.EXCLUDE_PATHS)); this.mergePaths = split(args.getOption(OptionParserFactory.MERGE_PATHS)); + this.includeIndex = args.hasOption(OptionParserFactory.INCLUDE_INDEX); this.failOnError = args.hasOption(OptionParserFactory.FAIL_ON_ERROR); this.earlyShutdown = args.hasOption(OptionParserFactory.EARLY_SHUTDOWN); this.skipInitialization = args.hasOption(OptionParserFactory.SKIP_INIT); @@ -133,6 +136,10 @@ public class MigrationOptions { return skipNameCheck; } + public boolean isIncludeIndex() { + return includeIndex; + } + private void logOptions() { if (copyBinariesByReference) { log.info("DataStore needs to be shared with new repository"); @@ -180,6 +187,10 @@ public class MigrationOptions { log.info("Test for long-named nodes will be disabled"); } + if (includeIndex) { + log.info("Index data for the paths {} will be copied", (Object) includePaths); + } + log.info("Cache size: {} MB", cacheSizeInMB); } diff --git a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java index aec1218..6ea732e 100644 --- a/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java +++ b/oak-upgrade/src/main/java/org/apache/jackrabbit/oak/upgrade/cli/parser/OptionParserFactory.java @@ -74,6 +74,8 @@ public class OptionParserFactory { public static final String SKIP_NAME_CHECK = "skip-name-check"; + public static final String INCLUDE_INDEX = "include-index"; + public static OptionParser create() { OptionParser op = new OptionParser(); addUsageOptions(op); @@ -122,6 +124,7 @@ public class OptionParserFactory { .ofType(String.class); op.accepts(MERGE_PATHS, "Comma-separated list of paths to merge during copy.").withRequiredArg() .ofType(String.class); + op.accepts(INCLUDE_INDEX, "Copy index data for paths specified in the " + INCLUDE_PATHS + " option"); } private static void addVersioningOptions(OptionParser op) { diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java new file mode 100644 index 0000000..e42dba3 --- /dev/null +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/IncludeIndexTest.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.upgrade; + +import com.google.common.base.Joiner; +import org.apache.jackrabbit.oak.commons.IOUtils; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.upgrade.cli.AbstractOak2OakTest; +import org.apache.jackrabbit.oak.upgrade.cli.OakUpgrade; +import org.apache.jackrabbit.oak.upgrade.cli.container.BlobStoreContainer; +import org.apache.jackrabbit.oak.upgrade.cli.container.FileDataStoreContainer; +import org.apache.jackrabbit.oak.upgrade.cli.container.MongoNodeStoreContainer; +import org.apache.jackrabbit.oak.upgrade.cli.container.NodeStoreContainer; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.jcr.RepositoryException; +import java.io.IOException; + +import static junit.framework.TestCase.assertTrue; +import static org.apache.jackrabbit.oak.upgrade.cli.container.MongoNodeStoreContainer.isMongoAvailable; +import static org.junit.Assert.assertFalse; +import static org.junit.Assume.assumeTrue; + +public class IncludeIndexTest extends AbstractOak2OakTest { + + private static final Logger log = LoggerFactory.getLogger(IncludeIndexTest.class); + + private final BlobStoreContainer blob; + + private final NodeStoreContainer source; + + private final NodeStoreContainer destination; + + private NodeStore nodeStore; + + @Before + public void prepare() throws Exception { + NodeStore source = getSourceContainer().open(); + try { + initContent(source); + } finally { + getSourceContainer().close(); + } + + String[] args = getArgs(); + log.info("oak2oak {}", Joiner.on(' ').join(args)); + OakUpgrade.main(args); + + nodeStore = destination.open(); + } + + @After + public void clean() throws IOException { + IOUtils.closeQuietly(getDestinationContainer()); + getDestinationContainer().clean(); + getSourceContainer().clean(); + } + + public IncludeIndexTest() throws IOException { + assumeTrue(isMongoAvailable()); + blob = new FileDataStoreContainer(); + source = new MongoNodeStoreContainer(blob); + destination = new MongoNodeStoreContainer(blob); + } + + @Override + protected NodeStoreContainer getSourceContainer() { + return source; + } + + @Override + protected NodeStoreContainer getDestinationContainer() { + return destination; + } + + @Override + protected String[] getArgs() { + return new String[] { "--src-datastore", blob.getDescription(), "--copy-versions=false", "--skip-init", "--include-index", "--include-paths=/apps,/libs", source.getDescription(), destination.getDescription() }; + } + + @Test + public void validateMigration() throws RepositoryException, IOException { + assertNodeExists("/oak:index/nodetype/:index/nt%3Afile/libs/sling/xss/config.xml"); + assertNodeExists("/oak:index/uuid/:index/" + getUuid("/apps/repl/components/repl/repl.html/jcr:content")); + + assertNodeMissing("/oak:index/nodetype/:index/nt%3Afile/sling.css"); + assertNodeMissing("/oak:index/uuid/:index/" + getUuid("/index.html/jcr:content")); + } + + private String getUuid(String path) { + return getNode(path).getString("jcr:uuid"); + } + + private void assertNodeMissing(String path) { + assertFalse(getNode(path).exists()); + } + + private void assertNodeExists(String path) { + assertTrue(getNode(path).exists()); + } + + private NodeState getNode(String path) { + NodeState ns = nodeStore.getRoot(); + for (String element : PathUtils.elements(path)) { + ns = ns.getChildNode(element); + } + return ns; + } +} \ No newline at end of file diff --git a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java index 9eb8cde..ad921cd 100644 --- a/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java +++ b/oak-upgrade/src/test/java/org/apache/jackrabbit/oak/upgrade/cli/AbstractOak2OakTest.java @@ -115,7 +115,7 @@ public abstract class AbstractOak2OakTest { } } - private void initContent(NodeStore target) throws IOException, RepositoryException, CommitFailedException { + protected void initContent(NodeStore target) throws IOException, RepositoryException, CommitFailedException { NodeStore initialContent = testContent.open(); try { RepositorySidegrade sidegrade = new RepositorySidegrade(initialContent, target);