diff --git oak-run/README.md oak-run/README.md index 5ea0e3804c..0dc7b2ae43 100644 --- oak-run/README.md +++ oak-run/README.md @@ -136,7 +136,11 @@ Explore The 'explore' mode starts a desktop browser GUI based on java swing which allows for read-only browsing of an existing oak repository. - $ java -jar oak-run-*.jar explore /path/to/oak/repository [skip-size-check] + $ java -jar oak-run-*.jar explore /path/to/oak/repository [--skip-size-check] + +Microsoft Azure node stores are also supported using the following command. The secret key must be supplied as an environment variable `AZURE_SECRET_KEY`. + + $ java -jar oak-run-*.jar explore az:https://myaccount.blob.core.windows.net/container/repository [--skip-size-check] frozennoderefsbyscanning ------------------------ diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java new file mode 100644 index 0000000000..1ef43554eb --- /dev/null +++ oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AbstractSegmentTarExplorerBackend.java @@ -0,0 +1,329 @@ +package org.apache.jackrabbit.oak.explorer; + +import com.google.common.base.Function; +import com.google.common.collect.Iterators; +import com.google.common.collect.Maps; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.segment.RecordId; +import org.apache.jackrabbit.oak.segment.SegmentBlob; +import org.apache.jackrabbit.oak.segment.SegmentId; +import org.apache.jackrabbit.oak.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper; +import org.apache.jackrabbit.oak.segment.SegmentPropertyState; +import org.apache.jackrabbit.oak.segment.file.JournalEntry; +import org.apache.jackrabbit.oak.segment.file.JournalReader; +import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import java.io.IOException; +import java.util.AbstractMap; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static com.google.common.collect.Lists.newArrayList; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.Collections.reverseOrder; + +/** + * Abstraction for Segment-Tar based backends. + */ +public abstract class AbstractSegmentTarExplorerBackend implements ExplorerBackend { + protected ReadOnlyFileStore store; + protected Map> index; + + + public abstract void open() throws IOException; + + public void close() { + store.close(); + store = null; + index = null; + } + + abstract protected JournalFile getJournal(); + + public List readRevisions() { + JournalFile journal = getJournal(); + + if (!journal.exists()) { + return newArrayList(); + } + + List revs = newArrayList(); + JournalReader journalReader = null; + + try { + journalReader = new JournalReader(journal); + Iterator revisionIterator = Iterators.transform(journalReader, + new Function() { + @Override + public String apply(JournalEntry entry) { + return entry.getRevision(); + } + }); + + try { + revs = newArrayList(revisionIterator); + } finally { + journalReader.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (journalReader != null) { + journalReader.close(); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + return revs; + } + + public Map> getTarReaderIndex() { + return store.getTarReaderIndex(); + } + + public Map> getTarGraph(String file) throws IOException { + return store.getTarGraph(file); + } + + public List getTarFiles() { + List files = new ArrayList<>(store.getTarReaderIndex().keySet()); + files.sort(reverseOrder()); + return files; + } + + public void getGcRoots(UUID uuidIn, Map>> links) throws IOException { + Deque todos = new ArrayDeque(); + todos.add(uuidIn); + Set visited = newHashSet(); + while (!todos.isEmpty()) { + UUID uuid = todos.remove(); + if (!visited.add(uuid)) { + continue; + } + for (String f : getTarFiles()) { + Map> graph = store.getTarGraph(f); + for (Map.Entry> g : graph.entrySet()) { + if (g.getValue() != null && g.getValue().contains(uuid)) { + UUID uuidP = g.getKey(); + if (!todos.contains(uuidP)) { + todos.add(uuidP); + Set> deps = links.get(uuid); + if (deps == null) { + deps = newHashSet(); + links.put(uuid, deps); + } + deps.add(new AbstractMap.SimpleImmutableEntry( + uuidP, f)); + } + } + } + } + } + } + + public Set getReferencedSegmentIds() { + Set ids = newHashSet(); + + for (SegmentId id : store.getReferencedSegmentIds()) { + ids.add(id.asUUID()); + } + + return ids; + } + + public NodeState getHead() { + return store.getHead(); + } + + public NodeState readNodeState(String recordId) { + return store.getReader().readNode(RecordId.fromString(store.getSegmentIdProvider(), recordId)); + } + + public void setRevision(String revision) { + store.setRevision(revision); + } + + public boolean isPersisted(NodeState state) { + return state instanceof SegmentNodeState; + } + + public boolean isPersisted(PropertyState state) { + return state instanceof SegmentPropertyState; + } + + public String getRecordId(NodeState state) { + if (state instanceof SegmentNodeState) { + return getRecordId((SegmentNodeState) state); + } + + return null; + } + + public UUID getSegmentId(NodeState state) { + if (state instanceof SegmentNodeState) { + return getSegmentId((SegmentNodeState) state); + } + + return null; + } + + public String getRecordId(PropertyState state) { + if (state instanceof SegmentPropertyState) { + return getRecordId((SegmentPropertyState) state); + } + + return null; + } + + public UUID getSegmentId(PropertyState state) { + if (state instanceof SegmentPropertyState) { + return getSegmentId((SegmentPropertyState) state); + } + + return null; + } + + public String getTemplateRecordId(NodeState state) { + if (state instanceof SegmentNodeState) { + return getTemplateRecordId((SegmentNodeState) state); + } + + return null; + } + + public UUID getTemplateSegmentId(NodeState state) { + if (state instanceof SegmentNodeState) { + return getTemplateSegmentId((SegmentNodeState) state); + } + + return null; + } + + public String getFile(NodeState state) { + if (state instanceof SegmentNodeState) { + return getFile((SegmentNodeState) state); + } + + return null; + } + + public String getFile(PropertyState state) { + if (state instanceof SegmentPropertyState) { + return getFile((SegmentPropertyState) state); + } + + return null; + } + + public String getTemplateFile(NodeState state) { + if (state instanceof SegmentNodeState) { + return getTemplateFile((SegmentNodeState) state); + } + + return null; + } + + public Map getBulkSegmentIds(Blob blob) { + Map result = Maps.newHashMap(); + + for (SegmentId segmentId : SegmentBlob.getBulkSegmentIds(blob)) { + result.put(segmentId.asUUID(), getFile(segmentId)); + } + + return result; + } + + public String getPersistedCompactionMapStats() { + return ""; + } + + public boolean isExternal(Blob blob) { + if (blob instanceof SegmentBlob) { + return isExternal((SegmentBlob) blob); + } + + return false; + } + + private boolean isExternal(SegmentBlob blob) { + return blob.isExternal(); + } + + private String getRecordId(SegmentNodeState state) { + return state.getRecordId().toString(); + } + + private UUID getSegmentId(SegmentNodeState state) { + return state.getRecordId().getSegmentId().asUUID(); + } + + private String getRecordId(SegmentPropertyState state) { + return state.getRecordId().toString(); + } + + private UUID getSegmentId(SegmentPropertyState state) { + return state.getRecordId().getSegmentId().asUUID(); + } + + private String getTemplateRecordId(SegmentNodeState state) { + RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); + + if (recordId == null) { + return null; + } + + return recordId.toString(); + } + + private UUID getTemplateSegmentId(SegmentNodeState state) { + RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); + + if (recordId == null) { + return null; + } + + return recordId.getSegmentId().asUUID(); + } + + private String getFile(SegmentNodeState state) { + return getFile(state.getRecordId().getSegmentId()); + } + + private String getFile(SegmentPropertyState state) { + return getFile(state.getRecordId().getSegmentId()); + } + + private String getTemplateFile(SegmentNodeState state) { + RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); + + if (recordId == null) { + return null; + } + + return getFile(recordId.getSegmentId()); + } + + private String getFile(SegmentId segmentId) { + for (Map.Entry> nameToId : index.entrySet()) { + for (UUID uuid : nameToId.getValue()) { + if (uuid.equals(segmentId.asUUID())) { + return nameToId.getKey(); + } + } + } + return null; + } +} diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java new file mode 100644 index 0000000000..f32c074841 --- /dev/null +++ oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java @@ -0,0 +1,45 @@ +package org.apache.jackrabbit.oak.explorer; + +import com.google.common.io.Files; +import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; + +import java.io.IOException; + +import static org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils.newSegmentNodeStorePersistence; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; + +/** + * Backend using a a remote Azure Segment Store. + *

+ * The path must be in the form "{@code az:https://myaccount.blob.core.windows.net/container/repository}". + * The secret key must be supplied as an environment variable {@codeAZURE_SECRET_KEY} + */ +public class AzureSegmentStoreExplorerBackend extends AbstractSegmentTarExplorerBackend { + private final String path; + private SegmentNodeStorePersistence persistence; + + public AzureSegmentStoreExplorerBackend(String path) { + this.path = path; + } + + @Override + public void open() throws IOException { + this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path); + + try { + this.store = fileStoreBuilder(Files.createTempDir()) + .withCustomPersistence(persistence) + .buildReadOnly(); + } catch (InvalidFileStoreVersionException e) { + throw new IllegalStateException(e); + } + this.index = store.getTarReaderIndex(); + } + + protected JournalFile getJournal() { + return persistence.getJournalFile(); + } +} diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/Explorer.java oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/Explorer.java index 43d2491600..7388258ca8 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/Explorer.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/Explorer.java @@ -18,28 +18,24 @@ */ package org.apache.jackrabbit.oak.explorer; +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.apache.commons.io.IOUtils; + +import javax.swing.*; +import javax.swing.UIManager.LookAndFeelInfo; import java.awt.*; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.KeyEvent; -import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import javax.swing.*; -import javax.swing.UIManager.LookAndFeelInfo; - -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import joptsimple.OptionSpec; -import org.apache.commons.io.IOUtils; - /** * NodeStore explorer - * - * GUI based on Swing, for now it is tailored to the TarMK - * + *

+ * GUI based on Swing, for now it is tailored to the TarMK and Azure Segment Store. */ public class Explorer { @@ -60,7 +56,7 @@ public class Explorer { public static void main(String[] args) throws IOException { OptionParser parser = new OptionParser(); OptionSpec skipSizeCheck = parser.accepts("skip-size-check", "Don't compute the size of the records"); - OptionSpec nonOptions = parser.nonOptions().ofType(File.class); + OptionSpec nonOptions = parser.nonOptions("path to repository").ofType(String.class); OptionSet options = parser.parse(args); if (options.valuesOf(nonOptions).isEmpty()) { @@ -68,16 +64,20 @@ public class Explorer { System.exit(1); } - File path = options.valuesOf(nonOptions).get(0); - - ExplorerBackend backend = new SegmentTarExplorerBackend(path); + String path = options.valuesOf(nonOptions).get(0); + ExplorerBackend backend; + if (path.startsWith("az:")) { + backend = new AzureSegmentStoreExplorerBackend(path); + } else { + backend = new SegmentTarExplorerBackend(path); + } new Explorer(path, backend, options.has(skipSizeCheck)); } private final ExplorerBackend backend; - private Explorer(final File path, ExplorerBackend backend, final boolean skipSizeCheck) throws IOException { + private Explorer(final String path, ExplorerBackend backend, final boolean skipSizeCheck) throws IOException { this.backend = backend; javax.swing.SwingUtilities.invokeLater(new Runnable() { @@ -92,7 +92,7 @@ public class Explorer { }); } - private void createAndShowGUI(final File path, boolean skipSizeCheck) throws IOException { + private void createAndShowGUI(final String path, boolean skipSizeCheck) throws IOException { JTextArea log = new JTextArea(5, 20); log.setMargin(new Insets(5, 5, 5, 5)); log.setLineWrap(true); @@ -159,13 +159,7 @@ public class Explorer { menuRefs.addActionListener(new ActionListener() { @Override public void actionPerformed(ActionEvent ev) { - List tarFiles = new ArrayList(); - for (File f : path.listFiles()) { - if (f.getName().endsWith(".tar")) { - tarFiles.add(f.getName()); - } - } - + List tarFiles = backend.getTarFiles(); String s = (String) JOptionPane.showInputDialog(frame, "Choose a tar file", "Tar File Info", JOptionPane.PLAIN_MESSAGE, null, tarFiles.toArray(), diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java index 250f6ce7d0..1d77c4c05f 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java @@ -17,54 +17,24 @@ package org.apache.jackrabbit.oak.explorer; -import static com.google.common.collect.Lists.newArrayList; -import static com.google.common.collect.Sets.newHashSet; -import static java.util.Collections.reverseOrder; -import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; +import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; import java.io.File; import java.io.IOException; -import java.util.AbstractMap.SimpleImmutableEntry; -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.Deque; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.UUID; -import com.google.common.base.Function; -import com.google.common.collect.Iterators; -import com.google.common.collect.Maps; -import org.apache.jackrabbit.oak.api.Blob; -import org.apache.jackrabbit.oak.api.PropertyState; -import org.apache.jackrabbit.oak.segment.RecordId; -import org.apache.jackrabbit.oak.segment.SegmentBlob; -import org.apache.jackrabbit.oak.segment.SegmentId; -import org.apache.jackrabbit.oak.segment.SegmentNodeState; -import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper; -import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; -import org.apache.jackrabbit.oak.segment.SegmentPropertyState; -import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; -import org.apache.jackrabbit.oak.segment.file.JournalEntry; -import org.apache.jackrabbit.oak.segment.file.JournalReader; -import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore; -import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.jetbrains.annotations.Nullable; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; -class SegmentTarExplorerBackend implements ExplorerBackend { +/** + * Backend using a local SegmentTar Node store. + */ +class SegmentTarExplorerBackend extends AbstractSegmentTarExplorerBackend { private final File path; - private ReadOnlyFileStore store; - - private Map> index; - - SegmentTarExplorerBackend(File path) throws IOException { - this.path = path; + SegmentTarExplorerBackend(String path) throws IOException { + this.path = new File(path); } @Override @@ -78,311 +48,7 @@ class SegmentTarExplorerBackend implements ExplorerBackend { } @Override - public void close() { - store.close(); - store = null; - index = null; - } - - @Override - public List readRevisions() { - JournalFile journal = new LocalJournalFile(path, "journal.log"); - - if (!journal.exists()) { - return newArrayList(); - } - - List revs = newArrayList(); - JournalReader journalReader = null; - - try { - journalReader = new JournalReader(journal); - Iterator revisionIterator = Iterators.transform(journalReader, - new Function() { - @Nullable - @Override - public String apply(JournalEntry entry) { - return entry.getRevision(); - } - }); - - try { - revs = newArrayList(revisionIterator); - } finally { - journalReader.close(); - } - } catch (IOException e) { - e.printStackTrace(); - } finally { - try { - if (journalReader != null) { - journalReader.close(); - } - } catch (IOException e) { - e.printStackTrace(); - } - } - - return revs; - } - - @Override - public Map> getTarReaderIndex() { - return store.getTarReaderIndex(); - } - - @Override - public Map> getTarGraph(String file) throws IOException { - return store.getTarGraph(file); - } - - @Override - public List getTarFiles() { - List files = new ArrayList<>(store.getTarReaderIndex().keySet()); - files.sort(reverseOrder()); - return files; - } - - @Override - public void getGcRoots(UUID uuidIn, Map>> links) throws IOException { - Deque todos = new ArrayDeque(); - todos.add(uuidIn); - Set visited = newHashSet(); - while (!todos.isEmpty()) { - UUID uuid = todos.remove(); - if (!visited.add(uuid)) { - continue; - } - for (String f : getTarFiles()) { - Map> graph = store.getTarGraph(f); - for (Entry> g : graph.entrySet()) { - if (g.getValue() != null && g.getValue().contains(uuid)) { - UUID uuidP = g.getKey(); - if (!todos.contains(uuidP)) { - todos.add(uuidP); - Set> deps = links.get(uuid); - if (deps == null) { - deps = newHashSet(); - links.put(uuid, deps); - } - deps.add(new SimpleImmutableEntry( - uuidP, f)); - } - } - } - } - } - } - - @Override - public Set getReferencedSegmentIds() { - Set ids = newHashSet(); - - for (SegmentId id : store.getReferencedSegmentIds()) { - ids.add(id.asUUID()); - } - - return ids; - } - - @Override - public NodeState getHead() { - return store.getHead(); + protected JournalFile getJournal() { + return new LocalJournalFile(path, "journal.log"); } - - @Override - public NodeState readNodeState(String recordId) { - return store.getReader().readNode(RecordId.fromString(store.getSegmentIdProvider(), recordId)); - } - - @Override - public void setRevision(String revision) { - store.setRevision(revision); - } - - @Override - public boolean isPersisted(NodeState state) { - return state instanceof SegmentNodeState; - } - - @Override - public boolean isPersisted(PropertyState state) { - return state instanceof SegmentPropertyState; - } - - @Override - public String getRecordId(NodeState state) { - if (state instanceof SegmentNodeState) { - return getRecordId((SegmentNodeState) state); - } - - return null; - } - - @Override - public UUID getSegmentId(NodeState state) { - if (state instanceof SegmentNodeState) { - return getSegmentId((SegmentNodeState) state); - } - - return null; - } - - @Override - public String getRecordId(PropertyState state) { - if (state instanceof SegmentPropertyState) { - return getRecordId((SegmentPropertyState) state); - } - - return null; - } - - @Override - public UUID getSegmentId(PropertyState state) { - if (state instanceof SegmentPropertyState) { - return getSegmentId((SegmentPropertyState) state); - } - - return null; - } - - @Override - public String getTemplateRecordId(NodeState state) { - if (state instanceof SegmentNodeState) { - return getTemplateRecordId((SegmentNodeState) state); - } - - return null; - } - - @Override - public UUID getTemplateSegmentId(NodeState state) { - if (state instanceof SegmentNodeState) { - return getTemplateSegmentId((SegmentNodeState) state); - } - - return null; - } - - @Override - public String getFile(NodeState state) { - if (state instanceof SegmentNodeState) { - return getFile((SegmentNodeState) state); - } - - return null; - } - - @Override - public String getFile(PropertyState state) { - if (state instanceof SegmentPropertyState) { - return getFile((SegmentPropertyState) state); - } - - return null; - } - - @Override - public String getTemplateFile(NodeState state) { - if (state instanceof SegmentNodeState) { - return getTemplateFile((SegmentNodeState) state); - } - - return null; - } - - @Override - public Map getBulkSegmentIds(Blob blob) { - Map result = Maps.newHashMap(); - - for (SegmentId segmentId : SegmentBlob.getBulkSegmentIds(blob)) { - result.put(segmentId.asUUID(), getFile(segmentId)); - } - - return result; - } - - @Override - public String getPersistedCompactionMapStats() { - return ""; - } - - @Override - public boolean isExternal(Blob blob) { - if (blob instanceof SegmentBlob) { - return isExternal((SegmentBlob) blob); - } - - return false; - } - - private boolean isExternal(SegmentBlob blob) { - return blob.isExternal(); - } - - private String getRecordId(SegmentNodeState state) { - return state.getRecordId().toString(); - } - - private UUID getSegmentId(SegmentNodeState state) { - return state.getRecordId().getSegmentId().asUUID(); - } - - private String getRecordId(SegmentPropertyState state) { - return state.getRecordId().toString(); - } - - private UUID getSegmentId(SegmentPropertyState state) { - return state.getRecordId().getSegmentId().asUUID(); - } - - private String getTemplateRecordId(SegmentNodeState state) { - RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); - - if (recordId == null) { - return null; - } - - return recordId.toString(); - } - - private UUID getTemplateSegmentId(SegmentNodeState state) { - RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); - - if (recordId == null) { - return null; - } - - return recordId.getSegmentId().asUUID(); - } - - private String getFile(SegmentNodeState state) { - return getFile(state.getRecordId().getSegmentId()); - } - - private String getFile(SegmentPropertyState state) { - return getFile(state.getRecordId().getSegmentId()); - } - - private String getTemplateFile(SegmentNodeState state) { - RecordId recordId = SegmentNodeStateHelper.getTemplateId(state); - - if (recordId == null) { - return null; - } - - return getFile(recordId.getSegmentId()); - } - - private String getFile(SegmentId segmentId) { - for (Entry> nameToId : index.entrySet()) { - for (UUID uuid : nameToId.getValue()) { - if (uuid.equals(segmentId.asUUID())) { - return nameToId.getKey(); - } - } - } - return null; - } - }