diff --git oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java index 4a7fd80..24223cf 100644 --- oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java +++ oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java @@ -25,8 +25,8 @@ import static java.lang.Integer.getInteger; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.commons.io.FileUtils.byteCountToDisplaySize; import static org.apache.jackrabbit.oak.api.Type.STRING; -import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_MK; import static org.apache.jackrabbit.oak.commons.FixturesHelper.getFixtures; +import static org.apache.jackrabbit.oak.commons.FixturesHelper.Fixture.SEGMENT_MK; import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions; import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder; @@ -46,32 +46,38 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.FutureTask; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.io.ByteStreams; import org.apache.jackrabbit.oak.api.Blob; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions; import org.apache.jackrabbit.oak.segment.file.FileStore; +import org.apache.jackrabbit.oak.segment.file.FileStoreGCMonitor; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.EmptyHook; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.stats.Clock; import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.io.ByteStreams; + public class CompactionAndCleanupIT { private static final Logger log = LoggerFactory @@ -919,6 +925,113 @@ public class CompactionAndCleanupIT { } } + /** + * Test asserting OAK-4669: No new generation of tar should be created when the segments are the same + * and when various indices are created. + */ + @Test + public void concurrentWritesCleanupNoNewGen() throws Exception { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + StatisticsProvider statsProvider = new DefaultStatisticsProvider(executor); + final FileStoreGCMonitor fileStoreGCMonitor = new FileStoreGCMonitor(Clock.SIMPLE); + + File fileStoreFolder = getFileStoreFolder(); + + final FileStore fileStore = fileStoreBuilder(fileStoreFolder) + .withGCOptions(defaultGCOptions().setRetainedGenerations(2)) + .withGCMonitor(fileStoreGCMonitor) + .withStatisticsProvider(statsProvider) + .withMaxFileSize(1) + .build(); + + final SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + ExecutorService executorService = Executors.newFixedThreadPool(5); + + try { + Runnable concurrentWriteTask = new Runnable() { + public void run() { + try { + NodeBuilder builder = nodeStore.getRoot().builder(); + builder.setProperty("blob" + new Random().nextInt(), createBlob(nodeStore, 512 * 512)); + + nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + fileStore.flush(); + } catch (CommitFailedException e) { + // ignore + } catch (IOException e) { + // ignore + } + }; + }; + + for (int i = 0; i < 5; i++) { + executorService.execute(concurrentWriteTask); + } + + fileStore.cleanup(); + + for (String fileName : fileStoreFolder.list()) { + if (fileName.endsWith(".tar")) { + int pos = fileName.length() - "a.tar".length(); + char generation = fileName.charAt(pos); + assertTrue("Expected generation is 'a', but instead was: '" + generation + "' for file " + fileName, + generation == 'a'); + } + } + } finally { + new ExecutorCloser(executorService).close(); + fileStore.close(); + } + } + + @Test + public void concurrentWritesCleanupZeroReclaimedSize() throws Exception { + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + StatisticsProvider statsProvider = new DefaultStatisticsProvider(executor); + final FileStoreGCMonitor fileStoreGCMonitor = new FileStoreGCMonitor(Clock.SIMPLE); + + final FileStore fileStore = fileStoreBuilder(getFileStoreFolder()) + .withGCOptions(defaultGCOptions().setRetainedGenerations(2)) + .withGCMonitor(fileStoreGCMonitor) + .withStatisticsProvider(statsProvider) + .withMaxFileSize(1) + .build(); + + final SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build(); + ExecutorService executorService = Executors.newFixedThreadPool(100); + + try { + Runnable concurrentWriteTask = new Runnable() { + public void run() { + try { + NodeBuilder builder = nodeStore.getRoot().builder(); + builder.setProperty("blob" + new Random().nextInt(), createBlob(nodeStore, 25 * 25)); + + nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + fileStore.flush(); + } catch (CommitFailedException e) { + // ignore + } catch (IOException e) { + // ignore + } + }; + }; + + for (int i = 0; i < 100; i++) { + executorService.execute(concurrentWriteTask); + } + + Thread.sleep(100); + fileStore.cleanup(); + + assertTrue("Reclaimed size expected is 0, but instead was: " + fileStoreGCMonitor.getLastReclaimedSize(), + fileStoreGCMonitor.getLastReclaimedSize() == 0); + } finally { + new ExecutorCloser(executorService).close(); + fileStore.close(); + } + } + private static void addContent(NodeBuilder builder) { for (int k = 0; k < 10000; k++) { builder.setProperty(UUID.randomUUID().toString(), UUID.randomUUID().toString());