Index: oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/YetAnotherConsistencyChecker.java =================================================================== --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/YetAnotherConsistencyChecker.java (nonexistent) +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/YetAnotherConsistencyChecker.java (working copy) @@ -0,0 +1,278 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.segment.file.tooling; + +import static org.apache.jackrabbit.oak.api.Type.BINARIES; +import static org.apache.jackrabbit.oak.api.Type.BINARY; +import static org.apache.jackrabbit.oak.commons.PathUtils.concat; +import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashSet; +import java.util.Set; + +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.segment.SegmentBlob; +import org.apache.jackrabbit.oak.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.segment.SegmentNotFoundException; +import org.apache.jackrabbit.oak.segment.file.JournalEntry; +import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +class YetAnotherConsistencyChecker { + + private YetAnotherConsistencyChecker() { + // Prevent external instantiation + } + + static class RepositoryCheckResult { + + private final JournalEntry fullyConsistent; + + private final JournalEntry partiallyConsistent; + + private final Set inconsistentCheckpoints; + + private RepositoryCheckResult(JournalEntry fullyConsistent, JournalEntry partiallyConsistent, Set inconsistentCheckpoints) { + this.fullyConsistent = fullyConsistent; + this.partiallyConsistent = partiallyConsistent; + this.inconsistentCheckpoints = inconsistentCheckpoints; + } + + public JournalEntry getFullyConsistent() { + return fullyConsistent; + } + + public JournalEntry getPartiallyConsistent() { + return partiallyConsistent; + } + + public Set getInconsistentCheckpoints() { + return inconsistentCheckpoints; + } + + } + + static RepositoryCheckResult checkRepository(ReadOnlyFileStore fileStore, SegmentNodeStore nodeStore, Iterable journal, boolean binaries) { + Set inconsistentHeadPaths = new HashSet<>(); + + Set visitedCheckpoints = new HashSet<>(); + Set consistentCheckpoints = new HashSet<>(); + + JournalEntry partiallyConsistent = null; + Set partiallyInconsistentCheckpoints = null; + + JournalEntry fullyConsistent = null; + + for (JournalEntry entry : journal) { + fileStore.setRevision(entry.getRevision()); + + // Check the head + + { + NodeState head = nodeStore.getRoot(); + + String bad = null; + + // Check first the paths we already determined as corrupted. + + for (String path : inconsistentHeadPaths) { + if (!checkPath(head, path, binaries)) { + bad = path; + break; + } + } + + // If none of the well known paths fail, traverse the head and + // stop at the first inconsistency. + + if (bad == null) { + inconsistentHeadPaths.clear(); + bad = checkTree(head, "/", binaries); + } + + // If an inconsistency is found, save the result for later and + // skip to the next revision. + + if (bad != null) { + inconsistentHeadPaths.add(bad); + continue; + } + } + + // Check the checkpoints + + Set inconsistentCheckpoints = new HashSet<>(); + + for (String checkpoint : nodeStore.checkpoints()) { + if (visitedCheckpoints.add(checkpoint)) { + + // This is a new checkpoint. Check its consistency and save + // the result for later use. + + NodeState root = nodeStore.retrieve(checkpoint); + + if (root == null) { + inconsistentCheckpoints.add(checkpoint); + continue; + } + + String bad = checkTree(root, "/", binaries); + + if (bad != null) { + inconsistentCheckpoints.add(checkpoint); + } else { + consistentCheckpoints.add(checkpoint); + } + } else { + + // We already visited this checkpoint. Since checkpoints are + // immutable, we reuse the previously computed result. + + if (consistentCheckpoints.contains(checkpoint)) { + continue; + } + + inconsistentCheckpoints.add(checkpoint); + } + } + + // At this point, we know that the head is consistent. If all the + // checkpoints in this revision are consistent too, we found a full + // consistency. We can return. + + if (inconsistentCheckpoints.isEmpty()) { + fullyConsistent = entry; + break; + } + + // Some checkpoints are inconsistent, but the head is consistent. If + // this is the first (most up to date) partial consistency, save + // this result for later. + + if (partiallyConsistent == null) { + partiallyConsistent = entry; + partiallyInconsistentCheckpoints = inconsistentCheckpoints; + } + } + + return new RepositoryCheckResult( + fullyConsistent, + partiallyConsistent, + partiallyInconsistentCheckpoints + ); + } + + private static boolean checkPath(NodeState root, String path, boolean binaries) { + NodeState child = getNode(root, path); + + if (child.exists()) { + return checkNode(child, binaries); + } + + return true; + } + + private static String checkTree(NodeState root, String path, boolean binaries) { + if (!checkNode(root, binaries)) { + return path; + } + + try { + for (ChildNodeEntry e : root.getChildNodeEntries()) { + String invalid = checkTree(e.getNodeState(), concat(path, e.getName()), binaries); + if (invalid != null) { + return invalid; + } + } + return null; + } catch (SegmentNotFoundException e) { + return path; + } + } + + private static boolean checkNode(NodeState node, boolean binaries) { + try { + for (PropertyState property : node.getProperties()) { + if (!checkProperty(property, binaries)) { + return false; + } + } + return true; + } catch (SegmentNotFoundException e) { + return false; + } + } + + private static boolean checkProperty(PropertyState property, boolean binaries) { + try { + if (property.isArray()) { + checkArrayProperty(property, binaries); + } else { + checkScalarProperty(property, binaries); + } + return true; + } catch (SegmentNotFoundException | IOException e) { + return false; + } + } + + private static void checkScalarProperty(PropertyState property, boolean binaries) throws IOException { + property.getValue(property.getType()); + if (property.getType() == BINARY && binaries) { + checkBinary(property.getValue(BINARY)); + } + } + + private static void checkArrayProperty(PropertyState property, boolean binaries) throws IOException { + for (int i = 0; i < property.count(); i++) { + property.getValue(property.getType().getBaseType(), i); + } + if (property.getType() == BINARIES && binaries) { + for (Blob blob : property.getValue(BINARIES)) { + checkBinary(blob); + } + } + } + + private static void checkBinary(Blob blob) throws IOException { + if (isExternal(blob)) { + return; + } + try (InputStream s = blob.getNewStream()) { + byte[] buffer = new byte[8192]; + int n; + do { + n = s.read(buffer); + } while (n >= 0); + } + } + + private static boolean isExternal(Blob b) { + if (b instanceof SegmentBlob) { + return ((SegmentBlob) b).isExternal(); + } + return false; + } + +} Property changes on: oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/YetAnotherConsistencyChecker.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property