diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java index 3a09afc..acb2ba9 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/CheckCommand.java @@ -21,6 +21,7 @@ import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidF import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import joptsimple.ArgumentAcceptingOptionSpec; import joptsimple.OptionParser; @@ -65,10 +66,13 @@ class CheckCommand implements Command { , "A deep scan of the content tree, traversing every node, will be performed by default."); } + PrintWriter out = new PrintWriter(System.out, true); + PrintWriter err = new PrintWriter(System.err, true); + if (options.has(segment)) { SegmentUtils.check(dir, journalFileName, true, debugLevel, binLen); } else { - SegmentTarUtils.check(dir, journalFileName, true, debugLevel, binLen, options.has(ioStatistics)); + SegmentTarUtils.check(dir, journalFileName, true, debugLevel, binLen, options.has(ioStatistics), out, err); } } diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java index ba7c70a..b4964d0 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java @@ -24,6 +24,7 @@ import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreB import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.Date; import java.util.List; @@ -184,7 +185,8 @@ final class SegmentTarUtils { .run(); } - static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen, boolean ioStatistics) { + static void check(File dir, String journalFileName, boolean fullTraversal, long debugLevel, long binLen, + boolean ioStatistics, PrintWriter outWriter, PrintWriter errWriter) { Check.builder() .withPath(dir) .withJournal(journalFileName) @@ -192,6 +194,8 @@ final class SegmentTarUtils { .withDebugInterval(debugLevel) .withMinimumBinaryLength(binLen) .withIOStatistics(ioStatistics) + .withOutWriter(outWriter) + .withErrWriter(errWriter) .build() .run(); } diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java index 1866184..e75751e 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java @@ -35,6 +35,8 @@ import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.PrintWriter; +import java.text.MessageFormat; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -51,8 +53,6 @@ import org.apache.jackrabbit.oak.segment.file.JournalReader; import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Utility for checking the files of a @@ -75,13 +75,15 @@ public class ConsistencyChecker implements Closeable { } - private static final Logger LOG = LoggerFactory.getLogger(ConsistencyChecker.class); - private final StatisticsIOMonitor statisticsIOMonitor = new StatisticsIOMonitor(); private final ReadOnlyFileStore store; private final long debugInterval; + + private final PrintWriter outWriter; + + private final PrintWriter errWriter; /** * Run a consistency check. @@ -93,6 +95,10 @@ public class ConsistencyChecker implements Closeable { * @param debugInterval number of seconds between printing progress information to * the console during the full traversal phase. * @param binLen number of bytes to read from binary properties. -1 for all. + * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency + * check was performed + * @param outWriter text output stream writer + * @param errWriter text error stream writer * @throws IOException */ public static void checkConsistency( @@ -101,13 +107,15 @@ public class ConsistencyChecker implements Closeable { boolean fullTraversal, long debugInterval, long binLen, - boolean ioStatistics + boolean ioStatistics, + PrintWriter outWriter, + PrintWriter errWriter ) throws IOException, InvalidFileStoreVersionException { - print("Searching for last good revision in {}", journalFileName); try ( JournalReader journal = new JournalReader(new File(directory, journalFileName)); - ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics) + ConsistencyChecker checker = new ConsistencyChecker(directory, debugInterval, ioStatistics, outWriter, errWriter) ) { + checker.print("Searching for last good revision in {0}", journalFileName); Set badPaths = newHashSet(); String latestGoodRevision = null; int revisionCount = 0; @@ -115,39 +123,39 @@ public class ConsistencyChecker implements Closeable { while (journal.hasNext() && latestGoodRevision == null) { String revision = journal.next(); try { - print("Checking revision {}", revision); + checker.print("Checking revision {0}", revision); revisionCount++; String badPath = checker.check(revision, badPaths, binLen); if (badPath == null && fullTraversal) { badPath = checker.traverse(revision, binLen); } if (badPath == null) { - print("Found latest good revision {}", revision); - print("Searched through {} revisions", revisionCount); + checker.print("Found latest good revision {0}", revision); + checker.print("Searched through {0} revisions", revisionCount); latestGoodRevision = revision; } else { badPaths.add(badPath); - print("Broken revision {}", revision); + checker.print("Broken revision {0}", revision); } } catch (IllegalArgumentException e) { - print("Skipping invalid record id {}", revision); + checker.printError("Skipping invalid record id {0}", revision); } } if (ioStatistics) { - print( - "[I/O] Segment read operations: {}", + checker.print( + "[I/O] Segment read operations: {0}", checker.statisticsIOMonitor.ioOperations ); - print( - "[I/O] Segment bytes read: {} ({} bytes)", + checker.print( + "[I/O] Segment bytes read: {0} ({1} bytes)", humanReadableByteCount(checker.statisticsIOMonitor.bytesRead.get()), checker.statisticsIOMonitor.bytesRead ); } if (latestGoodRevision == null) { - print("No good revision found"); + checker.print("No good revision found"); } } } @@ -155,18 +163,25 @@ public class ConsistencyChecker implements Closeable { /** * Create a new consistency checker instance * - * @param directory directory containing the tar files + * @param directory directory containing the tar files * @param debugInterval number of seconds between printing progress information to * the console during the full traversal phase. + * @param ioStatistics if {@code true} prints I/O statistics gathered while consistency + * check was performed + * @param outWriter text output stream writer + * @param errWriter text error stream writer * @throws IOException */ - public ConsistencyChecker(File directory, long debugInterval, boolean ioStatistics) throws IOException, InvalidFileStoreVersionException { + public ConsistencyChecker(File directory, long debugInterval, boolean ioStatistics, PrintWriter outWriter, + PrintWriter errWriter) throws IOException, InvalidFileStoreVersionException { FileStoreBuilder builder = fileStoreBuilder(directory); if (ioStatistics) { builder.withIOMonitor(statisticsIOMonitor); } this.store = builder.buildReadOnly(); this.debugInterval = debugInterval; + this.outWriter = outWriter; + this.errWriter = errWriter; } /** @@ -191,7 +206,7 @@ public class ConsistencyChecker implements Closeable { private String checkPath(String path, long binLen) { try { - print("Checking {}", path); + print("Checking {0}", path); NodeState root = SegmentNodeStoreBuilders.builder(store).build().getRoot(); String parentPath = getParentPath(path); String name = getName(path); @@ -202,7 +217,7 @@ public class ConsistencyChecker implements Closeable { return traverse(parent, parentPath, false, binLen); } } catch (RuntimeException e) { - print("Error while checking {}: {}", path, e.getMessage()); + printError("Error while checking {0}: {1}", path, e.getMessage()); return path; } } @@ -211,7 +226,7 @@ public class ConsistencyChecker implements Closeable { private int propertyCount; /** - * Travers the given {@code revision} + * Traverse the given {@code revision} * @param revision revision to travers * @param binLen number of bytes to read from binary properties. -1 for all. */ @@ -222,20 +237,20 @@ public class ConsistencyChecker implements Closeable { propertyCount = 0; String result = traverse(SegmentNodeStoreBuilders.builder(store).build() .getRoot(), "/", true, binLen); - print("Checked {} nodes and {} properties", nodeCount, propertyCount); + print("Checked {0} nodes and {1} properties", nodeCount, propertyCount); return result; } catch (RuntimeException e) { - print("Error while traversing {}", revision, e.getMessage()); + printError("Error while traversing {0}", revision, e.getMessage()); return "/"; } } private String traverse(NodeState node, String path, boolean deep, long binLen) { try { - debug("Traversing {}", path); + debug("Traversing {0}", path); nodeCount++; for (PropertyState propertyState : node.getProperties()) { - debug("Checking {}/{}", path, propertyState); + debug("Checking {0}/{1}", path, propertyState); Type type = propertyState.getType(); if (type == BINARY) { traverse(propertyState.getValue(BINARY), binLen); @@ -260,7 +275,7 @@ public class ConsistencyChecker implements Closeable { } return null; } catch (RuntimeException | IOException e) { - print("Error while traversing {}: {}", path, e.getMessage()); + printError("Error while traversing {0}: {1}", path, e.getMessage()); return path; } } @@ -297,29 +312,37 @@ public class ConsistencyChecker implements Closeable { store.close(); } - private static void print(String format) { - LOG.info(format); + private void print(String format) { + outWriter.println(format); } - private static void print(String format, Object arg) { - LOG.info(format, arg); + private void print(String format, Object arg) { + outWriter.println(MessageFormat.format(format, arg)); } - private static void print(String format, Object arg1, Object arg2) { - LOG.info(format, arg1, arg2); + private void print(String format, Object arg1, Object arg2) { + outWriter.println(MessageFormat.format(format, arg1, arg2)); + } + + private void printError(String format, Object arg) { + errWriter.println(MessageFormat.format(format, arg)); + } + + private void printError(String format, Object arg1, Object arg2) { + errWriter.println(MessageFormat.format(format, arg1, arg2)); } private long ts; private void debug(String format, Object arg) { if (debug()) { - LOG.debug(format, arg); + print(format, arg); } } private void debug(String format, Object arg1, Object arg2) { if (debug()) { - LOG.debug(format, arg1, arg2); + print(format, arg1, arg2); } } @@ -339,6 +362,4 @@ public class ConsistencyChecker implements Closeable { return false; } } - - } diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java index fcb8c9e..1811acb 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Check.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import java.io.File; +import java.io.PrintWriter; import org.apache.jackrabbit.oak.segment.file.tooling.ConsistencyChecker; @@ -54,6 +55,10 @@ public class Check implements Runnable { private long minimumBinaryLength; private boolean ioStatistics; + + private PrintWriter outWriter; + + private PrintWriter errWriter; private Builder() { // Prevent external instantiation. @@ -137,6 +142,28 @@ public class Check implements Runnable { this.ioStatistics = ioStatistics; return this; } + + /** + * The text output stream writer used to print normal output. + * @param outWriter the output writer. + * @return this builder. + */ + public Builder withOutWriter(PrintWriter outWriter) { + this.outWriter = outWriter; + + return this; + } + + /** + * The text error stream writer used to print erroneous output. + * @param errWriter the error writer. + * @return this builder. + */ + public Builder withErrWriter(PrintWriter errWriter) { + this.errWriter = errWriter; + + return this; + } /** * Create an executable version of the {@link Check} command. @@ -162,6 +189,10 @@ public class Check implements Runnable { private final long minimumBinaryLength; private final boolean ioStatistics; + + private final PrintWriter outWriter; + + private final PrintWriter errWriter; private Check(Builder builder) { this.path = builder.path; @@ -170,12 +201,14 @@ public class Check implements Runnable { this.debugInterval = builder.debugInterval; this.minimumBinaryLength = builder.minimumBinaryLength; this.ioStatistics = builder.ioStatistics; + this.outWriter = builder.outWriter; + this.errWriter = builder.errWriter; } @Override public void run() { try { - ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics); + ConsistencyChecker.checkConsistency(path, journal, fullTraversal, debugInterval, minimumBinaryLength, ioStatistics, outWriter, errWriter); } catch (Exception e) { e.printStackTrace(); } diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java new file mode 100644 index 0000000..4e2b5c8 --- /dev/null +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tooling/CheckValidRepositoryTest.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment.file.tooling; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; +import java.util.Random; + +import org.apache.jackrabbit.oak.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.tool.Check; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Lists; + +/** + * Tests for {@link CheckCommand} + */ +public class CheckValidRepositoryTest { + private static final Logger log = LoggerFactory.getLogger(CheckValidRepositoryTest.class); + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + @Before + public void setup() throws Exception { + FileStore fileStore = FileStoreBuilder.fileStoreBuilder(temporaryFolder.getRoot()) + .withMaxFileSize(256) + .withSegmentCacheSize(64) + .build(); + + SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + NodeBuilder builder = nodeStore.getRoot().builder(); + + addChildWithBlobProperties(nodeStore, builder, "a", 5); + addChildWithBlobProperties(nodeStore, builder, "b", 10); + addChildWithBlobProperties(nodeStore, builder, "c", 15); + + addChildWithProperties(nodeStore, builder, "d", 5); + addChildWithProperties(nodeStore, builder, "e", 5); + addChildWithProperties(nodeStore, builder, "f", 5); + + nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + fileStore.close(); + } + + @Test + public void testSuccessfulCheckWithBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withFullTraversal(true) + .withDebugInterval(Long.MAX_VALUE) + .withMinimumBinaryLength(Long.MAX_VALUE) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 7 nodes and 45 properties")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("")); + } + + @Test + public void testSuccessfulCheckWithoutBinaryTraversal() throws Exception { + StringWriter strOut = new StringWriter(); + StringWriter strErr = new StringWriter(); + + PrintWriter outWriter = new PrintWriter(strOut, true); + PrintWriter errWriter = new PrintWriter(strErr, true); + + Check.builder() + .withPath(new File(temporaryFolder.getRoot().getAbsolutePath())) + .withJournal("journal.log") + .withFullTraversal(true) + .withDebugInterval(Long.MAX_VALUE) + .withMinimumBinaryLength(0L) + .withIOStatistics(true) + .withOutWriter(outWriter) + .withErrWriter(errWriter) + .build() + .run(); + + outWriter.close(); + errWriter.close(); + + assertExpectedOutput(strOut.toString(), Lists.newArrayList("Searched through 1 revisions", "Checked 7 nodes and 15 properties")); + assertExpectedOutput(strErr.toString(), Lists.newArrayList("")); + } + + private static void assertExpectedOutput(String message, List assertMessages) { + log.info("Assert message: {}", assertMessages); + log.info("Message logged: {}", message); + + + for (String msg : assertMessages) { + Assert.assertTrue(message.contains(msg)); + } + } + + private static void addChildWithBlobProperties(SegmentNodeStore nodeStore, NodeBuilder builder, String childName, + int propCount) throws IOException { + NodeBuilder child = builder.child(childName); + for (int i = 0; i < propCount; i++) { + child.setProperty(childName + i, nodeStore.createBlob(randomStream(i, 2000))); + } + } + + private static void addChildWithProperties(SegmentNodeStore nodeStore, NodeBuilder builder, String childName, + int propCount) throws IOException { + NodeBuilder child = builder.child(childName); + for (int i = 0; i < propCount; i++) { + child.setProperty(childName + i, childName + i); + } + } + + private static InputStream randomStream(int seed, int size) { + Random r = new Random(seed); + byte[] data = new byte[size]; + r.nextBytes(data); + return new ByteArrayInputStream(data); + } +}