diff --git oak-doc/src/site/markdown/nodestore/segment/overview.md oak-doc/src/site/markdown/nodestore/segment/overview.md index 9893ba4..ff866ae 100644 --- oak-doc/src/site/markdown/nodestore/segment/overview.md +++ oak-doc/src/site/markdown/nodestore/segment/overview.md @@ -531,7 +531,7 @@ This tool is the counterpart of `backup`. ### Check ``` -java -jar oak-run.jar check PATH [--journal JOURNAL] [--notify SECS] [--bin] [--io-stats] +java -jar oak-run.jar check PATH [--journal JOURNAL] [--notify SECS] [--bin] [--filter PATH1[,PATH2,..,PATHn]] [--io-stats] ``` The `check` tool inspects an existing Segment Store at `PATH` for eventual inconsistencies. @@ -550,6 +550,12 @@ If the `--bin` option is specified, the tool will scan the full content of binar If not specified, the binary properties will not be traversed. The `--bin` option has no effect on binary properties stored in an external Blob Store. +If the `--filter` option is specified, the tool will traverse only the absolute paths specified as arguments. +At least one argument is expected with this option; multiple arguments need to be comma-separated. +The paths will be traversed in the same order as they were specified. +If one of the paths is invalid, the consistency check will fail and the traversal will not continue for the rest of the paths. +If the option is not specified, the full traversal of the repository (rooted at `/`) will be performed. + If the `--io-stats` option is specified, the tool will print some statistics about the I/O operations performed during the execution of the check command. This option is optional and is disabled by default. diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java index e5f5ed6..7f13ae0 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java @@ -22,6 +22,8 @@ import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidF import java.io.File; import java.io.IOException; import java.io.PrintWriter; +import java.util.LinkedHashSet; +import java.util.Set; import joptsimple.ArgumentAcceptingOptionSpec; import joptsimple.OptionParser; @@ -43,6 +45,9 @@ class CheckCommand implements Command { .withRequiredArg().ofType(Long.class).defaultsTo(Long.MAX_VALUE); OptionSpec bin = parser.accepts("bin", "read the content of binary properties"); OptionSpec segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar"); + ArgumentAcceptingOptionSpec filter = parser.accepts( + "filter", "comma separated content paths to be checked") + .withRequiredArg().ofType(String.class).withValuesSeparatedBy(',').defaultsTo("/"); OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)"); OptionSet options = parser.parse(args); @@ -57,17 +62,17 @@ class CheckCommand implements Command { File dir = isValidFileStoreOrFail(new File(options.nonOptionArguments().get(0).toString())); String journalFileName = journal.value(options); long debugLevel = notify.value(options); - + Set filterPaths = new LinkedHashSet(filter.values(options)); + if (options.has(deep)) { printUsage(parser, err, "The --deep option was deprecated! Please do not use it in the future!" , "A deep scan of the content tree, traversing every node, will be performed by default."); } - if (options.has(segment)) { SegmentUtils.check(dir, journalFileName, debugLevel, options.has(bin)); } else { - SegmentTarUtils.check(dir, journalFileName, debugLevel, options.has(bin), options.has(ioStatistics), out, err); + SegmentTarUtils.check(dir, journalFileName, debugLevel, options.has(bin), filterPaths, options.has(ioStatistics), out, err); } } diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java index a0c8f66..a23499b 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java @@ -28,8 +28,8 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.Set; -import com.google.common.io.Closer; import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever; import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; @@ -52,6 +52,8 @@ import org.apache.jackrabbit.oak.segment.tool.Revisions; import org.apache.jackrabbit.oak.segment.tool.SegmentGraph; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import com.google.common.io.Closer; + final class SegmentTarUtils { private static final boolean TAR_STORAGE_MEMORY_MAPPED = Boolean.getBoolean("tar.memoryMapped"); @@ -185,13 +187,14 @@ final class SegmentTarUtils { .run(); } - static void check(File dir, String journalFileName, long debugLevel, boolean checkBinaries, boolean ioStatistics, + static void check(File dir, String journalFileName, long debugLevel, boolean checkBinaries, Set filterPaths, boolean ioStatistics, PrintWriter outWriter, PrintWriter errWriter) { Check.builder() .withPath(dir) .withJournal(journalFileName) .withDebugInterval(debugLevel) .withCheckBinaries(checkBinaries) + .withFilterPaths(filterPaths) .withIOStatistics(ioStatistics) .withOutWriter(outWriter) .withErrWriter(errWriter) diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java index 08f4fc6..bdf155c 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java @@ -98,7 +98,8 @@ public class ConsistencyChecker implements Closeable { * @param journalFileName name of the journal file containing the revision history * @param debugInterval number of seconds between printing progress information to * the console during the full traversal phase. - * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @param filterPaths collection of repository paths to be checked * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency * check was performed * @param outWriter text output stream writer @@ -110,6 +111,7 @@ public class ConsistencyChecker implements Closeable { String journalFileName, long debugInterval, boolean checkBinaries, + Set filterPaths, boolean ioStatistics, PrintWriter outWriter, PrintWriter errWriter @@ -127,7 +129,7 @@ public class ConsistencyChecker implements Closeable { try { revisionCount++; - String corruptPath = checker.checkRevision(revision, corruptPaths, "/", checkBinaries); + String corruptPath = checker.checkRevision(revision, corruptPaths, filterPaths, checkBinaries); if (corruptPath == null) { checker.print("Found latest good revision {0}", revision); @@ -190,23 +192,22 @@ public class ConsistencyChecker implements Closeable { /** - * Checks the consistency of the supplied {@code path} at the given {@code revision}, + * Checks the consistency of the supplied {@code paths} at the given {@code revision}, * starting first with already known {@code corruptPaths}. * * @param revision revision to be checked * @param corruptPaths already known corrupt paths from previous revisions - * @param path initial path from which to start the consistency check, + * @param paths paths on which to run consistency check, * provided there are no corrupt paths. * @param checkBinaries if {@code true} full content of binary properties will be scanned * @return {@code null}, if the content tree rooted at path is consistent * in this revision or the path of the first inconsistency otherwise. */ - public String checkRevision(String revision, Set corruptPaths, String path, boolean checkBinaries) { + public String checkRevision(String revision, Set corruptPaths, Set paths, boolean checkBinaries) { print("Checking revision {0}", revision); String result = null; try { - print("Checking {0}", path); store.setRevision(revision); NodeState root = SegmentNodeStoreBuilders.builder(store).build().getRoot(); @@ -222,14 +223,23 @@ public class ConsistencyChecker implements Closeable { nodeCount = 0; propertyCount = 0; - NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, path); - result = checkNodeAndDescendants(wrapper.node, wrapper.path, checkBinaries); - print("Checked {0} nodes and {1} properties", nodeCount, propertyCount); + for (String path : paths) { + print("Checking {0}", path); + + NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, path); + result = checkNodeAndDescendants(wrapper.node, wrapper.path, checkBinaries); + + if (result != null) { + break; + } + } return result; } catch (RuntimeException e) { printError("Error while traversing {0}: {1}", revision, e.getMessage()); - return path; + return "/"; + } finally { + print("Checked {0} nodes and {1} properties", nodeCount, propertyCount); } } @@ -313,7 +323,11 @@ public class ConsistencyChecker implements Closeable { String name = getName(path); NodeState parent = getNode(root, parentPath); - if (!denotesRoot(path) && parent.hasChildNode(name)) { + if (!denotesRoot(path)) { + if (!parent.hasChildNode(name)) { + throw new IllegalArgumentException("Invalid path: " + path); + } + return new NodeWrapper(parent.getChildNode(name), path); } else { return new NodeWrapper(parent, parentPath); diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java index 1125060..ec47cb9 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java @@ -22,6 +22,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; import java.io.PrintWriter; +import java.util.Set; import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker; @@ -51,6 +52,8 @@ public class Check implements Runnable { private long debugInterval = Long.MAX_VALUE; private boolean checkBinaries; + + private Set filterPaths; private boolean ioStatistics; @@ -111,6 +114,19 @@ public class Check implements Runnable { this.checkBinaries = checkBinaries; return this; } + + /** + * Content paths to be checked. This parameter is not required and + * defaults to "/". + * + * @param filterPaths + * paths to be checked + * @return this builder. + */ + public Builder withFilterPaths(Set filterPaths) { + this.filterPaths = filterPaths; + return this; + } /** * Instruct the command to print statistics about I/O operations @@ -168,6 +184,8 @@ public class Check implements Runnable { private final long debugInterval; private final boolean checkBinaries; + + private final Set filterPaths; private final boolean ioStatistics; @@ -180,6 +198,7 @@ public class Check implements Runnable { this.journal = builder.journal; this.debugInterval = builder.debugInterval; this.checkBinaries = builder.checkBinaries; + this.filterPaths = builder.filterPaths; this.ioStatistics = builder.ioStatistics; this.outWriter = builder.outWriter; this.errWriter = builder.errWriter; @@ -188,7 +207,7 @@ public class Check implements Runnable { @Override public void run() { try { - ConsistencyChecker.checkConsistency(path, journal, debugInterval, checkBinaries, ioStatistics, outWriter, errWriter); + ConsistencyChecker.checkConsistency(path, journal, debugInterval, checkBinaries, filterPaths, ioStatistics, outWriter, errWriter); } catch (Exception e) { e.printStackTrace(); } diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java index 3c30f79..3e04bb4 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java @@ -24,8 +24,10 @@ import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.LinkedHashSet; import java.util.List; import java.util.Random; +import java.util.Set; import org.apache.jackrabbit.oak.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; @@ -77,18 +79,22 @@ public class CheckValidRepositoryTest { } @Test - public void testSuccessfulCheckWithBinaryTraversal() throws Exception { + public void testSuccessfulFullCheckWithBinaryTraversal() throws Exception { StringWriter strOut = new StringWriter(); StringWriter strErr = new StringWriter(); PrintWriter outWriter = new PrintWriter(strOut, true); PrintWriter errWriter = new PrintWriter(strErr, true); + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/"); + Check.builder() .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) .withJournal("journal.log") .withDebugInterval(Long.MAX_VALUE) .withCheckBinaries(true) + .withFilterPaths(filterPaths) .withIOStatistics(true) .withOutWriter(outWriter) .withErrWriter(errWriter) @@ -103,17 +109,56 @@ public class CheckValidRepositoryTest { } @Test - public void testSuccessfulCheckWithoutBinaryTraversal() throws Exception { + public void testSuccessfulOnlyRootKidsCheckWithBinaryTraversalAndFilterPaths() throws Exception { StringWriter strOut = new StringWriter(); StringWriter strErr = new StringWriter(); PrintWriter outWriter = new PrintWriter(strOut, true); PrintWriter errWriter = new PrintWriter(strErr, true); + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/a"); + filterPaths.add("/b"); + filterPaths.add("/c"); + filterPaths.add("/d"); + filterPaths.add("/e"); + filterPaths.add("/f"); + Check.builder() .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) .withJournal("journal.log") .withDebugInterval(Long.MAX_VALUE) + .withCheckBinaries(true) + .withFilterPaths(filterPaths) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 6 nodes and 45 properties")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("")); + } + + @Test + public void testSuccessfulFullCheckWithoutBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/"); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withDebugInterval(Long.MAX_VALUE) + .withFilterPaths(filterPaths) .withIOStatistics(true) .withOutWriter(outWriter) .withErrWriter(errWriter) @@ -127,6 +172,101 @@ public class CheckValidRepositoryTest { assertExpectedOutput(strErr.toString(), Lists.newArrayList("")); } + @Test + public void testSuccessfulPartialCheckWithoutBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/a"); + filterPaths.add("/b"); + filterPaths.add("/d"); + filterPaths.add("/e"); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withDebugInterval(Long.MAX_VALUE) + .withFilterPaths(filterPaths) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 4 nodes and 10 properties")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("")); + } + + @Test + public void testUnsuccessfulPartialCheckWithoutBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/g"); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withDebugInterval(Long.MAX_VALUE) + .withFilterPaths(filterPaths) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Broken revision", "Checked 0 nodes and 0 properties", "No good revision found")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("Invalid path: /g")); + } + + @Test + public void testUnsuccessfulPartialCheckWithBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Set filterPaths = new LinkedHashSet<>(); + filterPaths.add("/a"); + filterPaths.add("/f"); + filterPaths.add("/g"); + filterPaths.add("/d"); + filterPaths.add("/e"); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withDebugInterval(Long.MAX_VALUE) + .withFilterPaths(filterPaths) + .withCheckBinaries(true) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Broken revision", "Checked 2 nodes and 10 properties", "No good revision found")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("Invalid path: /g")); + } + private static void assertExpectedOutput(String message, List assertMessages) { log.info("Assert message: {}", assertMessages); log.info("Message logged: {}", message);