diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java index aa30337..22a2d69 100644 --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java @@ -72,6 +72,7 @@ import org.apache.jackrabbit.oak.plugins.segment.compaction.DefaultCompactionStr import org.apache.jackrabbit.oak.plugins.segment.file.FileStore; import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.Builder; import org.apache.jackrabbit.oak.plugins.segment.file.FileStoreGCMonitor; +import org.apache.jackrabbit.oak.plugins.segment.file.FileStoreStatsMBean; import org.apache.jackrabbit.oak.plugins.segment.file.GCMonitorMBean; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; @@ -87,6 +88,7 @@ import org.apache.jackrabbit.oak.spi.whiteboard.CompositeRegistration; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardExecutor; import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.osgi.framework.Constants; import org.osgi.framework.ServiceRegistration; import org.osgi.service.component.ComponentContext; @@ -254,6 +256,9 @@ public class SegmentNodeStoreService extends ProxyNodeStore policy = ReferencePolicy.DYNAMIC, target = ONLY_STANDALONE_TARGET) private volatile BlobStore blobStore; + @Reference + private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP; + private ServiceRegistration storeRegistration; private ServiceRegistration providerRegistration; private Registration checkpointRegistration; @@ -263,6 +268,7 @@ public class SegmentNodeStoreService extends ProxyNodeStore private Registration segmentCacheMBean; private Registration stringCacheMBean; private Registration fsgcMonitorMBean; + private Registration fileStoreStatsMBean; private WhiteboardExecutor executor; private boolean customBlobStore; @@ -384,7 +390,8 @@ public class SegmentNodeStoreService extends ProxyNodeStore .withCacheSize(Integer.parseInt(cache)) .withMaxFileSize(Integer.parseInt(size)) .withMemoryMapping("64".equals(mode)) - .withGCMonitor(gcMonitor); + .withGCMonitor(gcMonitor) + .withStatisticsProvider(statisticsProvider); if (customBlobStore) { log.info("Initializing SegmentNodeStore with BlobStore [{}]", blobStore); store = storeBuilder.withBlobStore(blobStore).create(); @@ -467,6 +474,12 @@ public class SegmentNodeStoreService extends ProxyNodeStore CompactionStrategyMBean.TYPE, "Segment node store compaction strategy settings"); + fileStoreStatsMBean = registerMBean(whiteboard, + FileStoreStatsMBean.class, + store.getStats(), + FileStoreStatsMBean.TYPE, + "FileStore statistics"); + log.info("SegmentNodeStore initialized"); return true; } @@ -542,6 +555,10 @@ public class SegmentNodeStoreService extends ProxyNodeStore fsgcMonitorMBean.unregister(); fsgcMonitorMBean = null; } + if (fileStoreStatsMBean != null) { + fileStoreStatsMBean.unregister(); + fileStoreStatsMBean = null; + } if (executor != null) { executor.stop(); executor = null; diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java index bcfa286..efbe5a2 100644 --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java @@ -82,6 +82,7 @@ import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.gc.GCMonitor; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -209,6 +210,8 @@ public class FileStore implements SegmentStore { private final ReadWriteLock fileStoreLock = new ReentrantReadWriteLock(); + private final FileStoreStats stats; + /** * Create a new instance of a {@link Builder} for a file store. * @param directory directory where the tar files are stored @@ -230,6 +233,7 @@ public class FileStore implements SegmentStore { private int cacheSize; // 0 -> DEFAULT_MEMORY_CACHE_SIZE private boolean memoryMapping; private final LoggingGCMonitor gcMonitor = new LoggingGCMonitor(); + private StatisticsProvider statsProvider = StatisticsProvider.NOOP; private Builder(File directory) { this.directory = directory; @@ -311,6 +315,18 @@ public class FileStore implements SegmentStore { return this; } + + /** + * {@link StatisticsProvider} for collecting statistics related to FileStore + * @param statisticsProvider + * @return this instance + */ + @Nonnull + public Builder withStatisticsProvider(@Nonnull StatisticsProvider statisticsProvider) { + this.statsProvider = checkNotNull(statisticsProvider); + return this; + } + /** * Create a new {@link FileStore} instance with the settings specified in this * builder. If none of the {@code with} methods have been called before calling @@ -322,6 +338,7 @@ public class FileStore implements SegmentStore { *
  • cache size: 256MB
    • *
  • memory mapping: on for 64 bit JVMs off otherwise
    • *
  • whiteboard: none. No {@link GCMonitor} tracking
    • + *
  • statsProvider: StatisticsProvider.NOOP
    • * * * @return a new file store instance @@ -330,14 +347,15 @@ public class FileStore implements SegmentStore { @Nonnull public FileStore create() throws IOException { return new FileStore( - blobStore, directory, root, maxFileSize, cacheSize, memoryMapping, gcMonitor, false); + blobStore, directory, root, maxFileSize, cacheSize, memoryMapping, gcMonitor, statsProvider, false); } } @Deprecated public FileStore(BlobStore blobStore, File directory, int maxFileSizeMB, boolean memoryMapping) throws IOException { - this(blobStore, directory, EMPTY_NODE, maxFileSizeMB, 0, memoryMapping, GCMonitor.EMPTY, false); + this(blobStore, directory, EMPTY_NODE, maxFileSizeMB, 0, memoryMapping, + GCMonitor.EMPTY, StatisticsProvider.NOOP, false); } @Deprecated @@ -355,24 +373,28 @@ public class FileStore implements SegmentStore { @Deprecated public FileStore(File directory, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) throws IOException { - this(null, directory, EMPTY_NODE, maxFileSizeMB, cacheSizeMB, memoryMapping, GCMonitor.EMPTY, false); + this(null, directory, EMPTY_NODE, maxFileSizeMB, cacheSizeMB, memoryMapping, + GCMonitor.EMPTY, StatisticsProvider.NOOP, false); } @Deprecated FileStore(File directory, NodeState initial, int maxFileSize) throws IOException { - this(null, directory, initial, maxFileSize, -1, MEMORY_MAPPING_DEFAULT, GCMonitor.EMPTY, false); + this(null, directory, initial, maxFileSize, -1, MEMORY_MAPPING_DEFAULT, + GCMonitor.EMPTY, StatisticsProvider.NOOP, false); } @Deprecated public FileStore( BlobStore blobStore, final File directory, NodeState initial, int maxFileSizeMB, int cacheSizeMB, boolean memoryMapping) throws IOException { - this(blobStore, directory, initial, maxFileSizeMB, cacheSizeMB, memoryMapping, GCMonitor.EMPTY, false); + this(blobStore, directory, initial, maxFileSizeMB, cacheSizeMB, memoryMapping, + GCMonitor.EMPTY, StatisticsProvider.NOOP,false); } private FileStore( BlobStore blobStore, final File directory, NodeState initial, int maxFileSizeMB, - int cacheSizeMB, boolean memoryMapping, GCMonitor gcMonitor, boolean readonly) + int cacheSizeMB, boolean memoryMapping, GCMonitor gcMonitor, StatisticsProvider statsProvider, + boolean readonly) throws IOException { if (readonly) { @@ -394,6 +416,7 @@ public class FileStore implements SegmentStore { this.maxFileSize = maxFileSizeMB * MB; this.memoryMapping = memoryMapping; this.gcMonitor = gcMonitor; + this.stats = new FileStoreStats(statsProvider, this); if (readonly) { journalFile = new RandomAccessFile(new File(directory, @@ -514,6 +537,7 @@ public class FileStore implements SegmentStore { }); approximateSize = new AtomicLong(size()); + stats.initializeRepoSize(approximateSize.get()); } else { flushThread = null; compactionThread = null; @@ -701,6 +725,15 @@ public class FileStore implements SegmentStore { } } + public int readerCount(){ + fileStoreLock.readLock().lock(); + try { + return readers.size(); + } finally { + fileStoreLock.readLock().unlock(); + } + } + /** * Returns the number of segments in this TarMK instance. * @@ -744,6 +777,10 @@ public class FileStore implements SegmentStore { return estimate; } + public FileStoreStats getStats() { + return stats; + } + public void flush() throws IOException { synchronized (persistedHead) { RecordId before = persistedHead.get(); @@ -883,6 +920,7 @@ public class FileStore implements SegmentStore { cm.remove(cleanedIds); long finalSize = size(); approximateSize.set(finalSize); + stats.reclaimed(initialSize - finalSize); gcMonitor.cleaned(initialSize - finalSize, finalSize); gcMonitor.info("TarMK GC #{}: cleanup completed in {} ({} ms). Post cleanup size is {} ({} bytes)" + " and space reclaimed {} ({} bytes). Compaction map weight/depth is {}/{} ({} bytes/{}).", @@ -1235,7 +1273,10 @@ public class FileStore implements SegmentStore { if (size >= maxFileSize) { newWriter(); } - approximateSize.addAndGet(TarWriter.BLOCK_SIZE + length + TarWriter.getPaddingSize(length)); + + int delta = TarWriter.BLOCK_SIZE + length + TarWriter.getPaddingSize(length); + stats.addWrites(delta); + approximateSize.addAndGet(delta); } finally { fileStoreLock.writeLock().unlock(); } @@ -1243,7 +1284,9 @@ public class FileStore implements SegmentStore { private void newWriter() throws IOException { if (writer.isDirty()) { + long sizeBeforeClose = writeFile.length(); writer.close(); + stats.addWrites(writeFile.length() - sizeBeforeClose); List list = newArrayListWithCapacity(1 + readers.size()); @@ -1352,13 +1395,13 @@ public class FileStore implements SegmentStore { public ReadOnlyStore(File directory) throws IOException { super(null, directory, EMPTY_NODE, -1, 0, MEMORY_MAPPING_DEFAULT, - GCMonitor.EMPTY, true); + GCMonitor.EMPTY, StatisticsProvider.NOOP, true); } public ReadOnlyStore(File directory, BlobStore blobStore) throws IOException { super(blobStore, directory, EMPTY_NODE, -1, 0, - MEMORY_MAPPING_DEFAULT, GCMonitor.EMPTY, true); + MEMORY_MAPPING_DEFAULT, GCMonitor.EMPTY, StatisticsProvider.NOOP, true); } /** diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStats.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStats.java new file mode 100644 index 0000000..8b0024b --- /dev/null +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStats.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.segment.file; + +import javax.annotation.Nonnull; +import javax.management.openmbean.CompositeData; + +import org.apache.jackrabbit.api.stats.TimeSeries; +import org.apache.jackrabbit.oak.commons.IOUtils; +import org.apache.jackrabbit.oak.stats.CounterStats; +import org.apache.jackrabbit.oak.stats.MeterStats; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; + +import static org.apache.jackrabbit.stats.TimeSeriesStatsUtil.asCompositeData; + +public class FileStoreStats implements FileStoreStatsMBean { + public static final String SEGMENT_REPO_SIZE = "SEGMENT_REPO_SIZE"; + public static final String SEGMENT_WRITES = "SEGMENT_WRITES"; + private final StatisticsProvider statisticsProvider; + private final FileStore store; + private final MeterStats writeStats; + private final CounterStats repoSize; + + public FileStoreStats(StatisticsProvider statisticsProvider, FileStore store) { + this.statisticsProvider = statisticsProvider; + this.store = store; + this.writeStats = statisticsProvider.getMeter(SEGMENT_WRITES); + this.repoSize = statisticsProvider.getCounterStats(SEGMENT_REPO_SIZE); + } + + void addWrites(long delta) { + writeStats.mark(delta); + repoSize.inc(delta); + } + + void initializeRepoSize(long size) { + repoSize.inc(size); + } + + void reclaimed(long size) { + repoSize.dec(size); + } + + //~--------------------------------< FileStoreStatsMBean > + + public long getSize() { + return store.size(); + } + + public int getTarFileCount() { + return store.readerCount() + 1; //1 for the writer + } + + @Nonnull + @Override + public CompositeData getWriteStats() { + return asCompositeData(getTimeSeries(SEGMENT_WRITES), SEGMENT_WRITES); + } + + @Nonnull + @Override + public CompositeData getRepositorySize() { + return asCompositeData(getTimeSeries(SEGMENT_REPO_SIZE), SEGMENT_REPO_SIZE); + } + + @Override + public String fileStoreInfoAsString() { + return String.format("Segment store size : %s%n" + + "Number of tar files : %d", + IOUtils.humanReadableByteCount(getSize()), + getTarFileCount()); + } + + private TimeSeries getTimeSeries(String name) { + return statisticsProvider.getStats().getTimeSeries(name, true); + } +} diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsMBean.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsMBean.java new file mode 100644 index 0000000..218def1 --- /dev/null +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreStatsMBean.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.segment.file; + +import javax.annotation.Nonnull; +import javax.management.openmbean.CompositeData; + +public interface FileStoreStatsMBean { + + String TYPE = "FileStoreStats"; + + long getSize(); + + int getTarFileCount(); + + /** + * @return time series of the writes to repository + */ + @Nonnull + CompositeData getWriteStats(); + + /** + * @return time series of the writes to repository + */ + @Nonnull + CompositeData getRepositorySize(); + + String fileStoreInfoAsString(); +} diff --git oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreServiceTest.java oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreServiceTest.java index e69fbc6..c7cd82b 100644 --- oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreServiceTest.java +++ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreServiceTest.java @@ -29,7 +29,9 @@ import java.util.Map; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.sling.testing.mock.osgi.junit.OsgiContext; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -43,6 +45,11 @@ public class SegmentNodeStoreServiceTest { @Rule public TemporaryFolder folder = new TemporaryFolder(); + @Before + public void setUp(){ + context.registerService(StatisticsProvider.class, StatisticsProvider.NOOP); + } + /** * A NodeStore service should be registered when a BlobStore service is not * available and the "customBlobStore" configuration property is false.