diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java index 3a09afc..50112da 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java @@ -35,14 +35,14 @@ class CheckCommand implements Command { ArgumentAcceptingOptionSpec journal = parser.accepts( "journal", "journal file") .withRequiredArg().ofType(String.class).defaultsTo("journal.log"); - OptionSpec deep = parser.accepts( - "deep", " enable deep consistency checking. "); + OptionSpec deep = parser.accepts( + "deep", " enable deep consistency checking."); ArgumentAcceptingOptionSpec notify = parser.accepts( "notify", "number of seconds between progress notifications") .withRequiredArg().ofType(Long.class).defaultsTo(Long.MAX_VALUE); - OptionSpec bin = parser.accepts("bin", "read the content of binary properties"); - OptionSpec segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar"); - OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)"); + OptionSpec bin = parser.accepts("bin", "read the content of binary properties"); + OptionSpec segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar"); + OptionSpec ioStatistics = parser.accepts("io-stats", "Print I/O statistics (only for oak-segment-tar)"); OptionSet options = parser.parse(args); @@ -53,22 +53,16 @@ class CheckCommand implements Command { File dir = isValidFileStoreOrFail(new File(options.nonOptionArguments().get(0).toString())); String journalFileName = journal.value(options); long debugLevel = notify.value(options); - - long binLen = 0L; - if (options.has(bin)) { - binLen = -1L; - } - if (options.has(deep)) { printUsage(parser, "The --deep option was deprecated! Please do not use it in the future!" , "A deep scan of the content tree, traversing every node, will be performed by default."); } if (options.has(segment)) { - SegmentUtils.check(dir, journalFileName, true, debugLevel, binLen); + SegmentUtils.check(dir, journalFileName, debugLevel, options.has(bin)); } else { - SegmentTarUtils.check(dir, journalFileName, true, debugLevel, binLen, options.has(ioStatistics)); + SegmentTarUtils.check(dir, journalFileName, debugLevel, options.has(bin), options.has(ioStatistics)); } } diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java index ba7c70a..8a5cfae 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java @@ -184,13 +184,12 @@ final class SegmentTarUtils { .run(); } - static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen, boolean ioStatistics) { + static void check(File dir, String journalFileName, long debugLevel, boolean checkBinaries, boolean ioStatistics) { Check.builder() .withPath(dir) .withJournal(journalFileName) - .withFullTraversal(fullTraversal) .withDebugInterval(debugLevel) - .withMinimumBinaryLength(binLen) + .withCheckBinaries(checkBinaries) .withIOStatistics(ioStatistics) .build() .run(); diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java index 964cf4e..77b77f8 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java @@ -162,8 +162,8 @@ class SegmentUtils { } } - static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen) throws IOException, InvalidFileStoreVersionException { - checkConsistency(dir, journalFileName, fullTraversal, debugLevel, binLen); + static void check(File dir, String journalFileName, long debugLevel, boolean checkBin) throws IOException, InvalidFileStoreVersionException { + checkConsistency(dir, journalFileName, true, debugLevel, checkBin ? -1L : 0L); } static void compact(File directory, boolean force) throws IOException, InvalidFileStoreVersionException { diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java index 1866184..4c4b7a5 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java @@ -20,7 +20,6 @@ package org.apache.jackrabbit.oak.segment.file.tooling; import static com.google.common.collect.Sets.newHashSet; -import static java.lang.Math.min; import static org.apache.jackrabbit.oak.api.Type.BINARIES; import static org.apache.jackrabbit.oak.api.Type.BINARY; import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount; @@ -82,25 +81,29 @@ public class ConsistencyChecker implements Closeable { private final ReadOnlyFileStore store; private final long debugInterval; + + private int nodeCount; + + private int propertyCount; + /** - * Run a consistency check. + * Run a full traversal consistency check. * * @param directory directory containing the tar files * @param journalFileName name of the journal file containing the revision history - * @param fullTraversal full traversal consistency check if {@code true}. Only try - * to access the root node otherwise. * @param debugInterval number of seconds between printing progress information to * the console during the full traversal phase. - * @param binLen number of bytes to read from binary properties. -1 for all. + * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency + * check was performed * @throws IOException */ public static void checkConsistency( File directory, String journalFileName, - boolean fullTraversal, long debugInterval, - long binLen, + boolean checkBinaries, boolean ioStatistics ) throws IOException, InvalidFileStoreVersionException { print("Searching for last good revision in {}", journalFileName); @@ -108,25 +111,24 @@ public class ConsistencyChecker implements Closeable { JournalReader journal = new JournalReader(new File(directory, journalFileName)); ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics) ) { - Set badPaths = newHashSet(); + Set corruptPaths = newHashSet(); + String initialPath = "/"; String latestGoodRevision = null; int revisionCount = 0; while (journal.hasNext() && latestGoodRevision == null) { String revision = journal.next(); try { - print("Checking revision {}", revision); revisionCount++; - String badPath = checker.check(revision, badPaths, binLen); - if (badPath == null && fullTraversal) { - badPath = checker.traverse(revision, binLen); - } - if (badPath == null) { + + String corruptPath = checker.checkRevision(revision, corruptPaths, initialPath, checkBinaries); + + if (corruptPath == null) { print("Found latest good revision {}", revision); print("Searched through {} revisions", revisionCount); latestGoodRevision = revision; } else { - badPaths.add(badPath); + corruptPaths.add(corruptPath); print("Broken revision {}", revision); } } catch (IllegalArgumentException e) { @@ -169,68 +171,60 @@ public class ConsistencyChecker implements Closeable { this.debugInterval = debugInterval; } + /** - * Check whether the nodes and all its properties of all given - * {@code paths} are consistent at the given {@code revision}. - * - * @param revision revision to check - * @param paths paths to check - * @param binLen number of bytes to read from binary properties. -1 for all. - * @return Path of the first inconsistency detected or {@code null} if none. + * Checks the consistency of the supplied {@code path} at the given {@code revision}, + * starting first with already known {@code corruptPaths}. + * + * @param revision revision to be checked + * @param corruptPaths already known corrupt paths from previous revisions + * @param path initial path from which to start the consistency check, + * provided there are no corrupt paths. + * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @return {@code null}, if the content tree rooted at path is consistent + * in this revision or the path of the first inconsistency otherwise. */ - public String check(String revision, Set paths, long binLen) { - store.setRevision(revision); - for (String path : paths) { - String err = checkPath(path, binLen); - if (err != null) { - return err; - } - } - return null; - } - - private String checkPath(String path, long binLen) { + public String checkRevision(String revision, Set corruptPaths, String path, boolean checkBinaries) { + print("Checking revision {}", revision); + String result = null; + try { - print("Checking {}", path); + store.setRevision(revision); NodeState root = SegmentNodeStoreBuilders.builder(store).build().getRoot(); - String parentPath = getParentPath(path); - String name = getName(path); - NodeState parent = getNode(root, parentPath); - if (!denotesRoot(path) && parent.hasChildNode(name)) { - return traverse(parent.getChildNode(name), path, false, binLen); - } else { - return traverse(parent, parentPath, false, binLen); - } - } catch (RuntimeException e) { - print("Error while checking {}: {}", path, e.getMessage()); - return path; - } - } - private int nodeCount; - private int propertyCount; + for (String corruptPath : corruptPaths) { + NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, corruptPath); + result = checkNode(wrapper.node, wrapper.path, checkBinaries); + + if (result != null) { + return result; + } + } - /** - * Travers the given {@code revision} - * @param revision revision to travers - * @param binLen number of bytes to read from binary properties. -1 for all. - */ - public String traverse(String revision, long binLen) { - try { - store.setRevision(revision); nodeCount = 0; propertyCount = 0; - String result = traverse(SegmentNodeStoreBuilders.builder(store).build() - .getRoot(), "/", true, binLen); + + NodeWrapper wrapper = NodeWrapper.deriveTraversableNodeOnPath(root, path); + result = checkNodeAndDescendants(wrapper.node, wrapper.path, checkBinaries); print("Checked {} nodes and {} properties", nodeCount, propertyCount); + return result; } catch (RuntimeException e) { - print("Error while traversing {}", revision, e.getMessage()); - return "/"; + print("Error while traversing {}: {}", revision, e.getMessage()); + return path; } } - - private String traverse(NodeState node, String path, boolean deep, long binLen) { + + /** + * Checks the consistency of a node and its properties at the given path. + * + * @param node node to be checked + * @param path path of the node + * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @return {@code null}, if the node is consistent, + * or the path of the first inconsistency otherwise. + */ + private String checkNode(NodeState node, String path, boolean checkBinaries) { try { debug("Traversing {}", path); nodeCount++; @@ -238,44 +232,85 @@ public class ConsistencyChecker implements Closeable { debug("Checking {}/{}", path, propertyState); Type type = propertyState.getType(); if (type == BINARY) { - traverse(propertyState.getValue(BINARY), binLen); + traverse(propertyState.getValue(BINARY), checkBinaries); } else if (type == BINARIES) { for (Blob blob : propertyState.getValue(BINARIES)) { - traverse(blob, binLen); + traverse(blob, checkBinaries); } } else { propertyCount++; propertyState.getValue(type); } } + + return null; + } catch (RuntimeException | IOException e) { + print("Error while traversing {}: {}", path, e.getMessage()); + return path; + } + } + + /** + * Recursively checks the consistency of a node and its descendants at the given path. + * @param node node to be checked + * @param path path of the node + * @param checkBinaries if {@code true} full content of binary properties will be scanned + * @return {@code null}, if the node is consistent, + * or the path of the first inconsistency otherwise. + */ + private String checkNodeAndDescendants(NodeState node, String path, boolean checkBinaries) { + String result = checkNode(node, path, checkBinaries); + if (result != null) { + return result; + } + + try { for (ChildNodeEntry cne : node.getChildNodeEntries()) { String childName = cne.getName(); NodeState child = cne.getNodeState(); - if (deep) { - String result = traverse(child, concat(path, childName), true, binLen); - if (result != null) { - return result; - } + result = checkNodeAndDescendants(child, concat(path, childName), checkBinaries); + if (result != null) { + return result; } } + return null; - } catch (RuntimeException | IOException e) { + } catch (RuntimeException e) { print("Error while traversing {}: {}", path, e.getMessage()); return path; } } - - private void traverse(Blob blob, long length) throws IOException { - if (length < 0) { - length = Long.MAX_VALUE; + + static class NodeWrapper { + NodeState node; + String path; + + NodeWrapper(NodeState node, String path) { + this.node = node; + this.path = path; + } + + static NodeWrapper deriveTraversableNodeOnPath(NodeState root, String path) { + String parentPath = getParentPath(path); + String name = getName(path); + NodeState parent = getNode(root, parentPath); + + if (!denotesRoot(path) && parent.hasChildNode(name)) { + return new NodeWrapper(parent.getChildNode(name), path); + } else { + return new NodeWrapper(parent, parentPath); + } } - if (length > 0 && !isExternal(blob)) { + } + + private void traverse(Blob blob, boolean checkBinaries) throws IOException { + if (checkBinaries && !isExternal(blob)) { InputStream s = blob.getNewStream(); try { byte[] buffer = new byte[8192]; - int l = s.read(buffer, 0, (int) min(buffer.length, length)); - while (l >= 0 && (length -= l) > 0) { - l = s.read(buffer, 0, (int) min(buffer.length, length)); + int l = s.read(buffer, 0, buffer.length); + while (l >= 0) { + l = s.read(buffer, 0, buffer.length); } } finally { s.close(); diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java index fcb8c9e..7b2d0d5 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java @@ -47,11 +47,9 @@ public class Check implements Runnable { private String journal; - private boolean fullTraversal; - private long debugInterval = Long.MAX_VALUE; - private long minimumBinaryLength; + private boolean checkBinaries; private boolean ioStatistics; @@ -83,19 +81,6 @@ public class Check implements Runnable { } /** - * Should a full traversal of the segment store be performed? This - * parameter is not required and defaults to {@code false}. - * - * @param fullTraversal {@code true} if a full traversal should be - * performed, {@code false} otherwise. - * @return this builder. - */ - public Builder withFullTraversal(boolean fullTraversal) { - this.fullTraversal = fullTraversal; - return this; - } - - /** * Number of seconds between successive debug print statements. This * parameter is not required and defaults to an arbitrary large number. * @@ -110,17 +95,15 @@ public class Check implements Runnable { } /** - * Minimum amount of bytes to read from binary properties. This - * parameter is not required and defaults to zero. + * Instruct the command to scan the full content of binary properties. + * This parameter is not required and defaults to {@code false}. * - * @param minimumBinaryLength minimum amount of bytes to read from - * binary properties. If this parameter is - * set to {@code -1}, every binary property - * is read in its entirety. + * @param checkBinaries {@code true} if binary properties should be + * scanned, {@code false} otherwise. * @return this builder. */ - public Builder withMinimumBinaryLength(long minimumBinaryLength) { - this.minimumBinaryLength = minimumBinaryLength; + public Builder withCheckBinaries(boolean checkBinaries) { + this.checkBinaries = checkBinaries; return this; } @@ -155,27 +138,24 @@ public class Check implements Runnable { private final String journal; - private final boolean fullTraversal; - private final long debugInterval; - private final long minimumBinaryLength; + private final boolean checkBinaries; private final boolean ioStatistics; private Check(Builder builder) { this.path = builder.path; this.journal = builder.journal; - this.fullTraversal = builder.fullTraversal; this.debugInterval = builder.debugInterval; - this.minimumBinaryLength = builder.minimumBinaryLength; + this.checkBinaries = builder.checkBinaries; this.ioStatistics = builder.ioStatistics; } @Override public void run() { try { - ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics); + ConsistencyChecker.checkConsistency(path, journal, debugInterval, checkBinaries, ioStatistics); } catch (Exception e) { e.printStackTrace(); }