diff --git a/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureCachingPersistenceTest.java b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureCachingPersistenceTest.java new file mode 100644 index 0000000000..e775a32543 --- /dev/null +++ b/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureCachingPersistenceTest.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.segment.azure; + +import com.microsoft.azure.storage.blob.CloudBlobContainer; +import org.apache.jackrabbit.oak.segment.persistentcache.CachingPersistenceTest; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; +import org.junit.Before; +import org.junit.ClassRule; + +import java.net.URISyntaxException; + +public class AzureCachingPersistenceTest extends CachingPersistenceTest { + + @ClassRule + public static AzuriteDockerRule azurite = new AzuriteDockerRule(); + + private CloudBlobContainer container; + + private int index; + + @Before + @Override + public void setup() throws Exception { + index = 0; + container = azurite.getContainer("oak-test"); + super.setup(); + } + + protected SegmentNodeStorePersistence createBackendPersistence() throws URISyntaxException { + return new AzurePersistence(container.getDirectoryReference("oak-" + index++)); + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java index 59f63b56d9..f17e29b035 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java @@ -253,6 +253,12 @@ public class SegmentNodeStoreFactory { ) boolean splitPersistence() default false; + @AttributeDefinition( + name = "Cache persistence", + description = "Boolean value indicating that the local persisted cache should be used for the custom segment store" + ) + boolean cachePersistence() default false; + @AttributeDefinition( name = "Backup directory", description = "Directory (relative to current working directory) for storing repository backups. " + @@ -477,6 +483,11 @@ public class SegmentNodeStoreFactory { return new File(getRepositoryHome(), appendRole("segmentstore-split")); } + @Override + public File getCachePersistenceDirectory() { + return new File(getRepositoryHome(), appendRole("segmentstore-cache")); + } + @Override public int getSegmentCacheSize() { return getCacheSize("segmentCache.size", configuration.segmentCache_size()); @@ -532,6 +543,11 @@ public class SegmentNodeStoreFactory { return configuration.splitPersistence(); } + @Override + public boolean hasCachePersistence() { + return configuration.cachePersistence(); + } + @Override public boolean registerDescriptors() { return configuration.registerDescriptors(); diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java index 5ea98317c0..e2a040fb73 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java @@ -59,6 +59,8 @@ import org.apache.jackrabbit.oak.segment.file.FileStoreStatsMBean; import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException; import org.apache.jackrabbit.oak.segment.file.MetricsIOMonitor; import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; +import org.apache.jackrabbit.oak.segment.persistentcache.CachingPersistence; +import org.apache.jackrabbit.oak.segment.persistentcache.DiskCache; import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; import org.apache.jackrabbit.oak.segment.split.SplitPersistence; import org.apache.jackrabbit.oak.spi.blob.BlobStore; @@ -120,6 +122,8 @@ class SegmentNodeStoreRegistrar { File getSplitPersistenceDirectory(); + File getCachePersistenceDirectory(); + int getSegmentCacheSize(); int getStringCacheSize(); @@ -142,6 +146,8 @@ class SegmentNodeStoreRegistrar { boolean hasSplitPersistence(); + boolean hasCachePersistence(); + boolean registerDescriptors(); String getRepositoryHome(); @@ -234,16 +240,25 @@ class SegmentNodeStoreRegistrar { } if (cfg.hasCustomSegmentStore() && cfg.getSegmentNodeStorePersistence() != null) { + SegmentNodeStorePersistence customPersistence = cfg.getSegmentNodeStorePersistence(); + + if (cfg.hasCachePersistence()) { + cfg.getLogger().info("Using local cache for the custom persistence [{}]", customPersistence); + cfg.getCachePersistenceDirectory().mkdirs(); + DiskCache cache = new DiskCache(cfg.getCachePersistenceDirectory()); + registerCloseable(cache); + customPersistence = new CachingPersistence(cache, customPersistence); + } + if (cfg.hasSplitPersistence()) { - cfg.getLogger().info("Initializing SegmentNodeStore with custom persistence [{}] and local writes", cfg.getSegmentNodeStorePersistence()); + cfg.getLogger().info("Initializing SegmentNodeStore with custom persistence [{}] and local writes", customPersistence); cfg.getSplitPersistenceDirectory().mkdirs(); - SegmentNodeStorePersistence roPersistence = cfg.getSegmentNodeStorePersistence(); SegmentNodeStorePersistence rwPersistence = new TarPersistence(cfg.getSplitPersistenceDirectory()); - SegmentNodeStorePersistence persistence = new SplitPersistence(roPersistence, rwPersistence); + SegmentNodeStorePersistence persistence = new SplitPersistence(customPersistence, rwPersistence); builder.withCustomPersistence(persistence); } else { - cfg.getLogger().info("Initializing SegmentNodeStore with custom persistence [{}]", cfg.getSegmentNodeStorePersistence()); - builder.withCustomPersistence(cfg.getSegmentNodeStorePersistence()); + cfg.getLogger().info("Initializing SegmentNodeStore with custom persistence [{}]", customPersistence); + builder.withCustomPersistence(customPersistence); } } diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java index 4cc939e6de..8110ca6a6b 100644 --- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java @@ -251,6 +251,12 @@ public class SegmentNodeStoreService { ) boolean splitPersistence() default false; + @AttributeDefinition( + name = "Cache persistence", + description = "Boolean value indicating that the local persisted cache should be used for the custom segment store" + ) + boolean cachePersistence() default false; + @AttributeDefinition( name = "Backup directory", description = "Directory (relative to current working directory) for storing repository backups. " + @@ -413,6 +419,11 @@ public class SegmentNodeStoreService { return new File(getRepositoryHome(), "segmentstore-split"); } + @Override + public File getCachePersistenceDirectory() { + return new File(getRepositoryHome(), "segmentstore-cache"); + } + @Override public int getSegmentCacheSize() { Integer size = Integer.getInteger("segmentCache.size"); @@ -488,6 +499,11 @@ public class SegmentNodeStoreService { return configuration.splitPersistence(); } + @Override + public boolean hasCachePersistence() { + return configuration.cachePersistence(); + } + @Override public boolean registerDescriptors() { return true; diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingArchiveManager.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingArchiveManager.java new file mode 100644 index 0000000000..8ccf97d710 --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingArchiveManager.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.jackrabbit.oak.segment.persistentcache; + +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveReader; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.UUID; + +public class CachingArchiveManager implements SegmentArchiveManager { + + private final SegmentArchiveManager delegate; + + private final DiskCache diskCache; + + public CachingArchiveManager(DiskCache diskCache, SegmentArchiveManager delegate) { + this.delegate = delegate; + this.diskCache = diskCache; + } + + @Override + public @NotNull List listArchives() throws IOException { + return delegate.listArchives(); + } + + @Override + public @Nullable SegmentArchiveReader open(@NotNull String archiveName) throws IOException { + return new CachingSegmentArchiveReader(diskCache, delegate.open(archiveName)); + } + + @Override + public @Nullable SegmentArchiveReader forceOpen(String archiveName) throws IOException { + return new CachingSegmentArchiveReader(diskCache, delegate.forceOpen(archiveName)); + } + + @Override + public @NotNull SegmentArchiveWriter create(@NotNull String archiveName) throws IOException { + return new CachingSegmentArchiveWriter(diskCache, delegate.create(archiveName)); + } + + @Override + public boolean delete(@NotNull String archiveName) { + return delegate.delete(archiveName); + } + + @Override + public boolean renameTo(@NotNull String from, @NotNull String to) { + return delegate.renameTo(from, to); + } + + @Override + public void copyFile(@NotNull String from, @NotNull String to) throws IOException { + delegate.copyFile(from, to); + } + + @Override + public boolean exists(@NotNull String archiveName) { + return delegate.exists(archiveName); + } + + @Override + public void recoverEntries(@NotNull String archiveName, @NotNull LinkedHashMap entries) throws IOException { + delegate.recoverEntries(archiveName, entries); + } +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistence.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistence.java new file mode 100644 index 0000000000..51c3c8132c --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistence.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.segment.persistentcache; + +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor; +import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.ManifestFile; +import org.apache.jackrabbit.oak.segment.spi.persistence.RepositoryLock; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; + +import java.io.IOException; + +public class CachingPersistence implements SegmentNodeStorePersistence { + + private final SegmentNodeStorePersistence delegate; + + private final DiskCache diskCache; + + public CachingPersistence(DiskCache diskCache, SegmentNodeStorePersistence delegate) { + this.delegate = delegate; + this.diskCache = diskCache; + } + + @Override + public SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException { + return new CachingArchiveManager(diskCache, delegate.createArchiveManager(memoryMapping, ioMonitor, fileStoreMonitor)); + } + + @Override + public boolean segmentFilesExist() { + return delegate.segmentFilesExist(); + } + + @Override + public JournalFile getJournalFile() { + return delegate.getJournalFile(); + } + + @Override + public GCJournalFile getGCJournalFile() throws IOException { + return delegate.getGCJournalFile(); + } + + @Override + public ManifestFile getManifestFile() throws IOException { + return delegate.getManifestFile(); + } + + @Override + public RepositoryLock lockRepository() throws IOException { + return delegate.lockRepository(); + } + +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveReader.java new file mode 100644 index 0000000000..473cad688b --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveReader.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.jackrabbit.oak.segment.persistentcache; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveEntry; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveReader; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * michid document + */ +public class CachingSegmentArchiveReader implements SegmentArchiveReader { + + @NotNull + private final DiskCache diskCache; + + @NotNull + private final SegmentArchiveReader delegate; + + public CachingSegmentArchiveReader( + @NotNull DiskCache diskCache, + @NotNull SegmentArchiveReader delegate) { + this.diskCache = diskCache; + this.delegate = delegate; + } + + @Override + @Nullable + public ByteBuffer readSegment(long msb, long lsb) throws IOException { + ByteBuffer buffer = diskCache.readSegment(msb, lsb); + if (buffer == null) { + buffer = delegate.readSegment(msb, lsb); + if (buffer != null) { + diskCache.writeSegment(msb, lsb, buffer); + } + } + return buffer; + } + + @Override + public boolean containsSegment(long msb, long lsb) { + if (diskCache.containsSegment(msb, lsb)) { + return true; + } else { + return delegate.containsSegment(msb, lsb); + } + } + + @Override + public List listSegments() { + return delegate.listSegments(); + } + + @Override + @Nullable + public ByteBuffer getGraph() throws IOException { + return delegate.getGraph(); + } + + @Override + public boolean hasGraph() { + return delegate.hasGraph(); + } + + @Override + @NotNull + public ByteBuffer getBinaryReferences() throws IOException { + return delegate.getBinaryReferences(); + } + + @Override + public long length() { + return delegate.length(); + } + + @Override + @NotNull + public String getName() { + return delegate.getName(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public int getEntrySize(int size) { + return delegate.getEntrySize(size); + } +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveWriter.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveWriter.java new file mode 100644 index 0000000000..4ea2b594ca --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingSegmentArchiveWriter.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.jackrabbit.oak.segment.persistentcache; + +import java.io.IOException; +import java.nio.ByteBuffer; + +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * michid document + */ +public class CachingSegmentArchiveWriter implements SegmentArchiveWriter { + + @NotNull + private final DiskCache diskCache; + + @NotNull + private final SegmentArchiveWriter delegate; + + public CachingSegmentArchiveWriter( + @NotNull DiskCache diskCache, + @NotNull SegmentArchiveWriter delegate) { + this.diskCache = diskCache; + this.delegate = delegate; + } + + @Override + public void writeSegment( + long msb, long lsb, @NotNull byte[] data, int offset, int size, + int generation, int fullGeneration, boolean isCompacted) + throws IOException { + delegate.writeSegment(msb, lsb, data, offset, size, generation, fullGeneration, isCompacted); + diskCache.writeSegment(msb, lsb, data, offset, size); + } + + @Override + @Nullable + public ByteBuffer readSegment(long msb, long lsb) throws IOException { + ByteBuffer buffer = diskCache.readSegment(msb, lsb); + if (buffer == null) { + buffer = delegate.readSegment(msb, lsb); + if (buffer != null) { + diskCache.writeSegment(msb, lsb, buffer); + } + } + return buffer; + } + + @Override + public boolean containsSegment(long msb, long lsb) { + if (diskCache.containsSegment(msb, lsb)) { + return true; + } else { + return delegate.containsSegment(msb, lsb); + } + } + + @Override + public void writeGraph(@NotNull byte[] data) throws IOException { + delegate.writeGraph(data); + } + + @Override + public void writeBinaryReferences(@NotNull byte[] data) throws IOException { + delegate.writeBinaryReferences(data); + } + + @Override + public long getLength() { + return delegate.getLength(); + } + + @Override + public int getEntryCount() { + return delegate.getEntryCount(); + } + + @Override + public void close() throws IOException { + delegate.close(); + } + + @Override + public boolean isCreated() { + return delegate.isCreated(); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + @Override + @NotNull + public String getName() { + return delegate.getName(); + } +} diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/DiskCache.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/DiskCache.java new file mode 100644 index 0000000000..5938fc9bcc --- /dev/null +++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/persistentcache/DiskCache.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.jackrabbit.oak.segment.persistentcache; + +import static java.util.Collections.emptySet; +import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.UUID; + +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.segment.file.tar.TarFiles; +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.jetbrains.annotations.NotNull; + +/** + * michid document + */ +public class DiskCache implements Closeable{ + + @NotNull + private final TarFiles tarFiles; + + public DiskCache(@NotNull TarFiles tarFiles) { + this.tarFiles = tarFiles; + } + + public DiskCache(@NotNull File directory) throws IOException { + this(TarFiles.builder() + .withDirectory(directory) + .withMaxFileSize(FileStoreBuilder.DEFAULT_MAX_FILE_SIZE * 1024 * 1024) + .withFileStoreMonitor(new FileStoreMonitorAdapter()) + .withIOMonitor(new IOMonitorAdapter()) + .withMemoryMapping(false) + .withTarRecovery((uuid, data, entryRecovery) -> { }) + .build()); + } + + public ByteBuffer readSegment(long msb, long lsb) { + return tarFiles.readSegment(msb, lsb); + } + + public boolean containsSegment(long msb, long lsb) { + return tarFiles.containsSegment(msb, lsb); + } + + public void writeSegment(long msb, long lsb, byte[] data, int offset, int size) throws IOException { + tarFiles.writeSegment( + new UUID(msb, lsb), data, offset, size, + newGCGeneration(0, 0, false), + emptySet(), emptySet()); // michid skip auxiliary entries for now. Segment graph is not needed. Binary references only for BlobGC. + } + + public void writeSegment(long msb, long lsb, ByteBuffer buffer) throws IOException { + if (buffer.hasArray()) { + writeSegment(msb, lsb, buffer.array(), buffer.arrayOffset(), buffer.remaining()); + } else { + ByteBuffer dup = buffer.duplicate(); + byte[] data = new byte[dup.remaining()]; + dup.get(data); + writeSegment(msb, lsb, data, 0, data.length); + } + } + + @Override + public void close() throws IOException { + tarFiles.close(); + } + +} diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistenceTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistenceTest.java new file mode 100644 index 0000000000..38937635d2 --- /dev/null +++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/persistentcache/CachingPersistenceTest.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.jackrabbit.oak.segment.persistentcache; + +import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence; +import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitorAdapter; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveReader; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter; +import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Random; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class CachingPersistenceTest { + + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private SegmentNodeStorePersistence cachingPersistence; + + private SegmentNodeStorePersistence backendPersistence; + + private DiskCache cache; + + @Before + public void setup() throws Exception { + backendPersistence = createBackendPersistence(); + cache = new DiskCache(folder.newFolder()); + cachingPersistence = new CachingPersistence(cache, backendPersistence); + } + + protected SegmentNodeStorePersistence createBackendPersistence() throws Exception { + return new TarPersistence(folder.newFolder()); + } + + @After + public void tearDown() throws IOException { + cache.close(); + } + + @Test + public void testWrittenSegmentsAddedToCache() throws Exception { + byte[] data = getRandomData(1024); + SegmentArchiveWriter writer = create(cachingPersistence, "data00000.tar"); + try { + writer.writeSegment(1, 1, data, 0, data.length, 0, 0, false); + } finally { + writer.close(); + } + + cleanBackendPersistence(); + + try (SegmentArchiveReader reader = open(cachingPersistence, "data00000.tar")) { + assertTrue(reader.containsSegment(1, 1)); + assertEquals(ByteBuffer.wrap(data), reader.readSegment(1, 1)); + } + } + + @Test + public void testReadSegmentsAddedToCache() throws Exception { + byte[] data = getRandomData(1024); + SegmentArchiveWriter writer = create(backendPersistence, "data00000.tar"); + try { + writer.writeSegment(1, 1, data, 0, data.length, 0, 0, false); + } finally { + writer.close(); + } + + try (SegmentArchiveReader reader = open(cachingPersistence, "data00000.tar")) { + reader.readSegment(1, 1); + } + + cleanBackendPersistence(); + + try (SegmentArchiveReader reader = open(cachingPersistence, "data00000.tar")) { + assertTrue(reader.containsSegment(1, 1)); + assertEquals(ByteBuffer.wrap(data), reader.readSegment(1, 1)); + } + } + + @Test + public void testWrittenSegmentsArePersisted() throws IOException { + byte[] data = getRandomData(1024); + SegmentArchiveWriter writer = create(cachingPersistence, "data00000.tar"); + try { + writer.writeSegment(1, 1, data, 0, data.length, 0, 0, false); + } finally { + writer.close(); + } + + try (SegmentArchiveReader reader = open(backendPersistence, "data00000.tar")) { + assertTrue(reader.containsSegment(1, 1)); + assertEquals(ByteBuffer.wrap(data), reader.readSegment(1, 1)); + } + } + + // use the old cache with a new backend, to see if the segments will be read from the cache + private void cleanBackendPersistence() throws Exception { + List archives = getManager(backendPersistence).listArchives(); + + // recreate the backend persistence with the same archives, but empty + backendPersistence = createBackendPersistence(); + for (String a : archives) { + SegmentArchiveWriter writer = create(backendPersistence, a); + writer.writeSegment(255, 255, new byte[10], 0, 10, 0, 0, false); + writer.close(); + } + cachingPersistence = new CachingPersistence(cache, backendPersistence); + } + + private static SegmentArchiveManager getManager(SegmentNodeStorePersistence persistence) throws IOException { + return persistence.createArchiveManager(true, new IOMonitorAdapter(), new FileStoreMonitorAdapter()); + } + + private static SegmentArchiveWriter create(SegmentNodeStorePersistence persistence, String name) throws IOException { + return getManager(persistence).create(name); + } + + private static SegmentArchiveReader open(SegmentNodeStorePersistence persistence, String name) throws IOException { + return getManager(persistence).open(name); + } + + private byte[] getRandomData(int length) { + byte[] data = new byte[length]; + new Random().nextBytes(data); + return data; + } +}