Index: oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java --- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java (revision 6a0e44dccde4b3aa5b18dc08f7b59eca1c62117e) +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java (date 1706890435050) @@ -19,20 +19,15 @@ package org.apache.jackrabbit.oak.segment.file; -import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; -import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThrows; -import static org.junit.Assert.fail; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; - import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.segment.LongIdMappingBlobStore; import org.apache.jackrabbit.oak.segment.SegmentId; import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder; import org.apache.jackrabbit.oak.segment.SegmentNodeState; +import org.apache.jackrabbit.oak.segment.SegmentNodeStore; +import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders; import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarManager; import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarWriter; import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; @@ -41,20 +36,38 @@ import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.jetbrains.annotations.NotNull; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; + +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; +import static org.junit.Assert.*; public class FileStoreTest { private static final String FAILED_TO_WRITE_ON_CLOSE = "Failed to write to the archive on closing"; + private static Logger LOG = LoggerFactory.getLogger(FileStoreTest.class); + @Rule public TemporaryFolder folder = new TemporaryFolder(new File("target")); - private File getFileStoreFolder() { - return folder.getRoot(); + private File getFileStoreFolder() throws IOException { + return folder.newFolder("segmentstore"); } @Test @@ -140,6 +153,70 @@ assertEquals("already shut down", closeEx.getMessage()); } + @Test + public void testRecovery_FileStore_withExternalBlobStore() throws InvalidFileStoreVersionException, IOException, CommitFailedException { + + File segmentStore = getFileStoreFolder(); + BlobStore blobStore = new LongIdMappingBlobStore(); + FileStore fileStore = createFileStore(segmentStore, blobStore); + + SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + + NodeBuilder builder = segmentNodeStore.getRoot().builder(); + builder.setProperty("foo", "bar"); + + Blob blob = builder.createBlob(new ZeroStream(100)); + builder.setProperty("binaryProperty", blob); + + segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + fileStore.flush(); + + // create another segment + builder.setProperty("foo1", "bar1"); + segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + fileStore.flush(); + + // copy all files form segmentStore to segmentStoreClone + // with this, tha last tar archive will not have index, graph and binary references + File segmentStoreClone = folder.newFolder("segmentstore-clone"); + Files.walk(segmentStore.toPath()) + .forEach(source -> { + try { + Path target = segmentStoreClone.toPath().resolve(segmentStore.toPath().relativize(source)); + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + }); + + + fileStore.close(); + + // Start new FileStore, which should be able to recover successfully despite now having the index, graph and binary references + try { + fileStore = createFileStore(segmentStoreClone, blobStore); + } catch (Exception e){ + fail("Should not throw exception" + e); + } + + segmentNodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + + + builder = segmentNodeStore.getRoot().builder(); + + assertNotNull(builder.getProperty("foo")); + assertNotNull(builder.getProperty("foo1")); + blob = builder.getProperty("binaryProperty").getValue(Type.BINARY); + assertNotNull(blob); + } + + public FileStore createFileStore(File segmentStore, BlobStore blobStore) throws IOException, InvalidFileStoreVersionException { + return fileStoreBuilder(segmentStore) + .withBinariesInlineThreshold(0) + .withBlobStore(blobStore) + .build(); + } + private static int counter = 0; private static void writeData(FileStore fileStore, int size) throws IOException { Index: oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java new file mode 100644 --- /dev/null (date 1706890435049) +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java (date 1706890435049) @@ -0,0 +1,72 @@ +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.guava.common.base.Strings; +import org.apache.jackrabbit.oak.spi.blob.BlobOptions; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +public abstract class IdMappingBlobStore implements BlobStore { + + private final MemoryBlobStore bs = new MemoryBlobStore(); + + private final Map ids = new HashMap<>(); + + @Override + public String writeBlob(InputStream inputStream) throws IOException { + String in = bs.writeBlob(inputStream); + String out = generateId(); + ids.put(out, in); + return out; + } + + @Override + public String writeBlob(InputStream inputStream, BlobOptions options) throws IOException { + return writeBlob(inputStream); + } + + @Override + public int readBlob(String s, long l, byte[] bytes, int i, int i1) throws IOException { + return bs.readBlob(mapId(s), l, bytes, i, i1); + } + + @Override + public long getBlobLength(String s) throws IOException { + return bs.getBlobLength(mapId(s)); + } + + @Override + public InputStream getInputStream(String s) throws IOException { + return bs.getInputStream(mapId(s)); + } + + @Override + public String getBlobId(@NotNull String s) { + return bs.getBlobId(s); + } + + @Override + public String getReference(@NotNull String s) { + return bs.getBlobId(mapId(s)); + } + + @Override + public void close() throws Exception { + bs.close(); + } + + private String mapId(String in) { + String out = ids.get(in); + if (out == null) { + throw new IllegalArgumentException("in"); + } + return out; + } + + protected abstract String generateId(); +} Index: oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java new file mode 100644 --- /dev/null (date 1706890435049) +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java (date 1706890435049) @@ -0,0 +1,14 @@ +package org.apache.jackrabbit.oak.segment; + +import org.apache.jackrabbit.guava.common.base.Strings; + +public class LongIdMappingBlobStore extends IdMappingBlobStore { + + private static int next; + + @Override + protected String generateId() { + return Strings.repeat("0", Segment.BLOB_ID_SMALL_LIMIT) + Integer.toString(next++); + } + +}