diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java index 090febfc8f..f254c529af 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java @@ -83,7 +83,8 @@ public class FileStoreBackupImpl implements FileStoreBackup { backup.getSegmentIdProvider(), backup.getBlobStore(), new WriterCacheManager.Default(), - bufferWriter + bufferWriter, + backup.getBinariesInlineThreshold() ); ClassicCompactor compactor = new ClassicCompactor( backup.getReader(), diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java index 261cebc050..dda8fddf0c 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java @@ -79,7 +79,8 @@ public class FileStoreRestoreImpl implements FileStoreRestore { store.getSegmentIdProvider(), store.getBlobStore(), new WriterCacheManager.Default(), - bufferWriter + bufferWriter, + store.getBinariesInlineThreshold() ); SegmentGCOptions gcOptions = defaultGCOptions().setOffline(); ClassicCompactor compactor = new ClassicCompactor( diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java index 0aedf2bef9..ed9453ec3b 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java @@ -112,19 +112,23 @@ public class DefaultSegmentWriter implements SegmentWriter { @NotNull private final WriteOperationHandler writeOperationHandler; + + private final int binariesInlineThreshold; /** * Create a new instance of a {@code SegmentWriter}. Note the thread safety * properties pointed out in the class comment. * - * @param store store to write to - * @param reader segment reader for the {@code store} - * @param idProvider segment id provider for the {@code store} - * @param blobStore the blog store or {@code null} for inlined - * blobs - * @param cacheManager cache manager instance for the - * de-duplication caches used by this writer - * @param writeOperationHandler handler for write operations. + * @param store store to write to + * @param reader segment reader for the {@code store} + * @param idProvider segment id provider for the {@code store} + * @param blobStore the blob store or {@code null} for inlined + * blobs + * @param cacheManager cache manager instance for the + * de-duplication caches used by this writer + * @param writeOperationHandler handler for write operations. + * @param binariesInlineThreshold threshold in bytes, specifying the limit up to which + * blobs will be inlined */ public DefaultSegmentWriter( @NotNull SegmentStore store, @@ -132,7 +136,8 @@ public class DefaultSegmentWriter implements SegmentWriter { @NotNull SegmentIdProvider idProvider, @Nullable BlobStore blobStore, @NotNull WriterCacheManager cacheManager, - @NotNull WriteOperationHandler writeOperationHandler + @NotNull WriteOperationHandler writeOperationHandler, + int binariesInlineThreshold ) { this.store = checkNotNull(store); this.reader = checkNotNull(reader); @@ -140,6 +145,9 @@ public class DefaultSegmentWriter implements SegmentWriter { this.blobStore = blobStore; this.cacheManager = checkNotNull(cacheManager); this.writeOperationHandler = checkNotNull(writeOperationHandler); + checkArgument(binariesInlineThreshold >= 0); + checkArgument(binariesInlineThreshold <= Segment.MEDIUM_LIMIT); + this.binariesInlineThreshold = binariesInlineThreshold; } @Override @@ -638,11 +646,13 @@ public class DefaultSegmentWriter implements SegmentWriter { } private RecordId internalWriteStream(@NotNull InputStream stream) throws IOException { - // Special case for short binaries (up to about 16kB): + // Special case for short binaries (up to about binariesInlineThreshold, 16kB by default): // store them directly as small- or medium-sized value records - byte[] data = new byte[Segment.MEDIUM_LIMIT]; + + byte[] data = new byte[binariesInlineThreshold]; int n = read(stream, data, 0, data.length); - if (n < Segment.MEDIUM_LIMIT) { + + if (n < binariesInlineThreshold) { return writeValueRecord(n, data); } @@ -651,6 +661,17 @@ public class DefaultSegmentWriter implements SegmentWriter { new ByteArrayInputStream(data, 0, n), stream)); return writeBlobId(blobId); } + + // handle case in which blob store is not configured and + // binariesInlineThreshold < Segment.MEDIUM_LIMIT + // store the binaries as small or medium sized value records + + data = Arrays.copyOf(data, Segment.MEDIUM_LIMIT); + n += read(stream, data, n, Segment.MEDIUM_LIMIT - n); + + if (n < Segment.MEDIUM_LIMIT) { + return writeValueRecord(n, data); + } data = Arrays.copyOf(data, Segment.MAX_SEGMENT_SIZE); n += read(stream, data, n, Segment.MAX_SEGMENT_SIZE - n); diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java index f1e2bcd98e..d0437d5a58 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java @@ -151,7 +151,8 @@ public final class DefaultSegmentWriterBuilder { store.getSegmentIdProvider(), store.getBlobStore(), cacheManager, - createWriter(store, pooled) + createWriter(store, pooled), + store.getBinariesInlineThreshold() ); } @@ -187,7 +188,8 @@ public final class DefaultSegmentWriterBuilder { public void flush(@NotNull SegmentStore store) { throw new UnsupportedOperationException("Cannot write to read-only store"); } - } + }, + store.getBinariesInlineThreshold() ); } @@ -202,7 +204,8 @@ public final class DefaultSegmentWriterBuilder { store.getSegmentIdProvider(), store.getBlobStore(), cacheManager, - createWriter(store, pooled) + createWriter(store, pooled), + Segment.MEDIUM_LIMIT ); } diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java index 484e207c79..7950025ec6 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java @@ -112,8 +112,8 @@ public class Segment { * value. And since small values are never stored as medium ones, we can * extend the size range to cover that many longer values. */ - static final int MEDIUM_LIMIT = (1 << (16 - 2)) + SMALL_LIMIT; - + public static final int MEDIUM_LIMIT = (1 << (16 - 2)) + SMALL_LIMIT; + /** * Maximum size of small blob IDs. A small blob ID is stored in a value * record whose length field contains the pattern "1110" in its most diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java index 719af35cc6..61cea135d1 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java @@ -130,6 +130,8 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable { protected final IOMonitor ioMonitor; protected final RemoteStoreMonitor remoteStoreMonitor; + + protected final int binariesInlineThreshold; AbstractFileStore(final FileStoreBuilder builder) { this.directory = builder.getDirectory(); @@ -153,6 +155,7 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable { this.ioMonitor = builder.getIOMonitor(); this.remoteStoreMonitor = builder.getRemoteStoreMonitor(); this.segmentBufferMonitor = new SegmentBufferMonitor(builder.getStatsProvider()); + this.binariesInlineThreshold = builder.getBinariesInlineThreshold(); } static SegmentNotFoundException asSegmentNotFoundException(Exception e, SegmentId id) { @@ -189,6 +192,10 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable { public SegmentIdProvider getSegmentIdProvider() { return tracker; } + + public int getBinariesInlineThreshold() { + return binariesInlineThreshold; + } /** * @return the {@link Revisions} object bound to the current store. diff --git oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java index b5ec98337c..28c883931c 100644 --- oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java +++ oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.segment.file; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Sets.newHashSet; @@ -37,10 +38,12 @@ import java.io.IOException; import java.util.Set; import com.google.common.base.Predicate; + import org.apache.jackrabbit.oak.segment.CacheWeights.NodeCacheWeigher; import org.apache.jackrabbit.oak.segment.CacheWeights.StringCacheWeigher; import org.apache.jackrabbit.oak.segment.CacheWeights.TemplateCacheWeigher; import org.apache.jackrabbit.oak.segment.RecordCache; +import org.apache.jackrabbit.oak.segment.Segment; import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener; import org.apache.jackrabbit.oak.segment.WriterCacheManager; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; @@ -66,7 +69,7 @@ import org.slf4j.LoggerFactory; */ public class FileStoreBuilder { private static final Logger LOG = LoggerFactory.getLogger(FileStore.class); - + private static final boolean MEMORY_MAPPING_DEFAULT = "64".equals(System.getProperty("sun.arch.data.model", "32")); @@ -95,6 +98,8 @@ public class FileStoreBuilder { private boolean memoryMapping = MEMORY_MAPPING_DEFAULT; private boolean offHeapAccess = getBoolean("access.off.heap"); + + private int binariesInlineThreshold = Segment.MEDIUM_LIMIT; private SegmentNodeStorePersistence persistence; @@ -396,6 +401,16 @@ public class FileStoreBuilder { this.eagerSegmentCaching = eagerSegmentCaching; return this; } + + /** + * Sets the threshold under which binaries are inlined in data segments. + * @param binariesInlineThreshold the threshold + * @return this instance + */ + public FileStoreBuilder withBinariesInlineThreshold(int binariesInlineThreshold) { + this.binariesInlineThreshold = binariesInlineThreshold; + return this; + } public Backend buildProcBackend(AbstractFileStore fileStore) throws IOException { return new FileStoreProcBackend(fileStore, persistence); @@ -574,6 +589,10 @@ public class FileStoreBuilder { boolean getEagerSegmentCaching() { return eagerSegmentCaching; } + + int getBinariesInlineThreshold() { + return binariesInlineThreshold; + } @Override public String toString() { diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java new file mode 100644 index 0000000000..4444055946 --- /dev/null +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/BinariesInlineThresholdIT.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.segment; + +import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.Random; + +import org.apache.jackrabbit.core.data.FileDataStore; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; +import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder; +import org.apache.jackrabbit.oak.spi.blob.BlobStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +public class BinariesInlineThresholdIT { + private static final int SMALL_BINARIES_INLINE_THRESHOLD = 1024; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(new File("target")); + + private BlobStore createBlobStore(File dir) { + FileDataStore fds = new FileDataStore(); + fds.setMinRecordLength(4096); + fds.init(dir.getAbsolutePath()); + return new DataStoreBlobStore(fds); + } + + private FileStore createFileStore(File dir, BlobStore blobStore, int binariesInlineThreshold) throws Throwable { + FileStoreBuilder builder = fileStoreBuilder(dir) + .withMaxFileSize(1) + .withMemoryMapping(false) + .withNodeDeduplicationCacheSize(1) + .withSegmentCacheSize(256) + .withStringCacheSize(0) + .withTemplateCacheSize(0) + .withBinariesInlineThreshold(binariesInlineThreshold); + + if (blobStore != null) { + builder.withBlobStore(blobStore); + } + + return builder.build(); + } + + @Test + public void testInlineBinaries() throws Throwable { + File fileStoreDir = folder.newFolder(); + File blobStoreDir = folder.newFolder(); + + BlobStore blobStore = createBlobStore(blobStoreDir); + + // set binaries inline threshold to default Segment.MEDIUM limit, i.e. 16512 + // all binaries under this limit should be inlined, if over the limit they + // will be stored in the blobstore + FileStore fileStore = createFileStore(fileStoreDir, blobStore, Segment.MEDIUM_LIMIT); + + SegmentNodeStore sns = SegmentNodeStoreBuilders.builder(fileStore).build(); + + Blob b1 = addTestContent(sns, "a", SMALL_BINARIES_INLINE_THRESHOLD - 1); + assertTrue(b1 instanceof SegmentBlob); + assertNull(((SegmentBlob) b1).getBlobId()); + assertFalse(((SegmentBlob) b1).isExternal()); + assertFalse(b1.isInlined()); + + Blob b2 = addTestContent(sns, "b", SMALL_BINARIES_INLINE_THRESHOLD); + assertTrue(b2 instanceof SegmentBlob); + assertNull(((SegmentBlob) b2).getBlobId()); + assertFalse(((SegmentBlob) b2).isExternal()); + assertFalse(b2.isInlined()); + + Blob b3 = addTestContent(sns, "c", Segment.MEDIUM_LIMIT - 1); + assertTrue(b3 instanceof SegmentBlob); + assertNull(((SegmentBlob) b3).getBlobId()); + assertFalse(((SegmentBlob) b3).isExternal()); + assertFalse(b3.isInlined()); + + Blob b4 = addTestContent(sns, "d", Segment.MEDIUM_LIMIT); + assertTrue(b4 instanceof SegmentBlob); + assertNotNull(b4.getReference()); + assertEquals(b4.getContentIdentity(), ((SegmentBlob) b4).getBlobId()); + assertFalse(b4.isInlined()); + + fileStore.close(); + + fileStore = createFileStore(fileStoreDir, blobStore, SMALL_BINARIES_INLINE_THRESHOLD); + sns = SegmentNodeStoreBuilders.builder(fileStore).build(); + + Blob b5 = addTestContent(sns, "e", SMALL_BINARIES_INLINE_THRESHOLD - 1); + assertTrue(b5 instanceof SegmentBlob); + assertNull(((SegmentBlob) b5).getBlobId()); + assertFalse(((SegmentBlob) b5).isExternal()); + assertFalse(b5.isInlined()); + + Blob b6 = addTestContent(sns, "f", SMALL_BINARIES_INLINE_THRESHOLD); + assertTrue(b6 instanceof SegmentBlob); + assertNull(b6.getReference()); + assertEquals(b6.getContentIdentity(), ((SegmentBlob) b6).getBlobId()); + assertTrue(b6.isInlined()); + + Blob b7 = addTestContent(sns, "g", Segment.MEDIUM_LIMIT - 1); + assertTrue(b7 instanceof SegmentBlob); + assertNotNull(b7.getReference()); + assertEquals(b7.getContentIdentity(), ((SegmentBlob) b7).getBlobId()); + assertFalse(b7.isInlined()); + + Blob b8 = addTestContent(sns, "h", Segment.MEDIUM_LIMIT); + assertTrue(b8 instanceof SegmentBlob); + assertNotNull(b8.getReference()); + assertEquals(b8.getContentIdentity(), ((SegmentBlob) b8).getBlobId()); + assertFalse(b8.isInlined()); + + fileStore.compactFull(); + + // b2 and b3 blobs should have ended now in blob store + // as a result of compaction rewriting the repository + // using SMALL_BINARIES_INLINE_THRESHOLD + + b2 = getBlob(sns, "b"); + assertTrue(b2 instanceof SegmentBlob); + assertNotNull(((SegmentBlob) b2).getBlobId()); + assertTrue(((SegmentBlob) b2).isExternal()); + assertTrue(b2.isInlined()); + + b3 = getBlob(sns, "c"); + assertTrue(b3 instanceof SegmentBlob); + assertNotNull(b3.getReference()); + assertEquals(b3.getContentIdentity(), ((SegmentBlob) b3).getBlobId()); + assertFalse(b3.isInlined()); + + blobStore.close(); + fileStore.close(); + } + + private Blob getBlob(SegmentNodeStore sns, String child) { + PropertyState ps = sns.getRoot().getChildNode(child).getProperty("testBlob"); + return ps.getValue(Type.BINARY); + } + + private Blob addTestContent(NodeStore store, String child, int size) throws CommitFailedException, IOException { + NodeBuilder builder = store.getRoot().builder(); + builder.child(child).setProperty("ts", System.currentTimeMillis()); + + byte[] data = new byte[size]; + new Random().nextBytes(data); + Blob blob = store.createBlob(new ByteArrayInputStream(data)); + + builder.child(child).setProperty("testBlob", blob); + + store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + return blob; + } +} diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java index f5b91a478d..db256b9438 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterTest.java @@ -29,10 +29,17 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import com.google.common.base.Charsets; -import com.google.common.base.Strings; -import com.google.common.collect.ImmutableMap; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.oak.segment.test.TemporaryFileStore; import org.jetbrains.annotations.NotNull; import org.junit.Before; @@ -43,13 +50,9 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import com.google.common.base.Charsets; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import ch.qos.logback.classic.Level; import ch.qos.logback.classic.LoggerContext; @@ -63,9 +66,11 @@ public class DefaultSegmentWriterTest { private final byte[] bytes = HELLO_WORLD.getBytes(Charsets.UTF_8); + private static final int SMALL_BINARIES_INLINE_THRESHOLD = 4; + private TemporaryFolder folder = new TemporaryFolder(new File("target")); - private TemporaryFileStore store = new TemporaryFileStore(folder, false); + private TemporaryFileStore store = new TemporaryFileStore(folder, SMALL_BINARIES_INLINE_THRESHOLD); @Rule public RuleChain rules = RuleChain.outerRule(folder).around(store); @@ -77,6 +82,14 @@ public class DefaultSegmentWriterTest { writer = defaultSegmentWriterBuilder("test").build(store.fileStore()); } + @Test + public void testValueRecord() throws IOException { + InputStream stream = new ByteArrayInputStream(bytes); + RecordId valueId = writer.writeStream(stream); + SegmentBlob blob = new SegmentBlob(null, valueId); + assertEquals(HELLO_WORLD, IOUtils.toString(blob.getNewStream(), Charsets.UTF_8)); + } + @Test public void testBlockRecord() throws IOException { RecordId blockId = writer.writeBlock(bytes, 0, bytes.length); diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java index ed9d4cce2e..78ca438f6a 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java @@ -98,6 +98,8 @@ public class SegmentDataStoreBlobGCIT { private DataStoreBlobStore blobStore; private SegmentGCOptions gcOptions = defaultGCOptions(); + + private int binariesInlineThreshold = Segment.MEDIUM_LIMIT; @Rule public TemporaryFolder folder = new TemporaryFolder(new File("target")); @@ -152,13 +154,13 @@ public class SegmentDataStoreBlobGCIT { NodeBuilder a = nodeStore.getRoot().builder(); - /* Create garbage by creating in-lined blobs (size < 16KB) */ + /* Create garbage by creating in-lined blobs (size < binaries inline threshold) */ int number = 500; NodeBuilder content = a.child("content"); for (int i = 0; i < number; i++) { NodeBuilder c = content.child("x" + i); for (int j = 0; j < 5; j++) { - c.setProperty("p" + j, nodeStore.createBlob(randomStream(j, 16384))); + c.setProperty("p" + j, nodeStore.createBlob(randomStream(j, binariesInlineThreshold - 1))); } } nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY); diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java index a57597e2d0..0d316abd70 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/standby/client/RemoteBlobProcessorTest.java @@ -52,6 +52,8 @@ public class RemoteBlobProcessorTest { }; private TemporaryFileStore fileStore = new TemporaryFileStore(folder, blobStore, false); + + private int binariesInlineThreshold = SegmentTestConstants.MEDIUM_LIMIT; @Rule public RuleChain rules = RuleChain.outerRule(folder) @@ -104,7 +106,7 @@ public class RemoteBlobProcessorTest { SegmentNodeStore store = SegmentNodeStoreBuilders.builder(fileStore.fileStore()).build(); NodeBuilder root = store.getRoot().builder(); - root.setProperty("b", root.createBlob(new NullInputStream(SegmentTestConstants.MEDIUM_LIMIT - 1))); + root.setProperty("b", root.createBlob(new NullInputStream(binariesInlineThreshold - 1))); store.merge(root, EmptyHook.INSTANCE, CommitInfo.EMPTY); RemoteBlobProcessor processor = new RemoteBlobProcessor(blobStore.blobStore(), blobId -> { diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java index 851cccefa8..434edf1401 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/test/TemporaryFileStore.java @@ -23,6 +23,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.segment.Segment; import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; import org.apache.jackrabbit.oak.segment.file.FileStore; @@ -43,6 +44,8 @@ public class TemporaryFileStore extends ExternalResource { private FileStore store; + private int binariesInlineThreshold = Segment.MEDIUM_LIMIT; + public TemporaryFileStore(TemporaryFolder folder, boolean standby) { this.folder = folder; this.standby = standby; @@ -55,12 +58,20 @@ public class TemporaryFileStore extends ExternalResource { this.standby = standby; } + public TemporaryFileStore(TemporaryFolder folder, int binariesInlineThreshold) { + this.folder = folder; + this.standby = false; + this.blobStore = null; + this.binariesInlineThreshold = binariesInlineThreshold; + } + @Override protected void before() throws Throwable { executor = Executors.newSingleThreadScheduledExecutor(); FileStoreBuilder builder = fileStoreBuilder(folder.newFolder()) .withMaxFileSize(1) .withMemoryMapping(false) + .withBinariesInlineThreshold(binariesInlineThreshold) .withNodeDeduplicationCacheSize(1) .withSegmentCacheSize(256) .withStringCacheSize(0)