From d055a6665d65a2c1f92abe7b7eaab3d572cec2a4 Mon Sep 17 00:00:00 2001 From: Robert Munteanu Date: Wed, 6 May 2015 18:33:39 +0300 Subject: [PATCH] OAK-2065 - JMX stats for operations being performed in DocumentStore --- .../oak/plugins/document/DocumentNodeStore.java | 119 +++++++++++++++++++-- .../plugins/document/DocumentNodeStoreMBean.java | 14 +++ .../plugins/document/DocumentNodeStoreService.java | 6 ++ 3 files changed, 129 insertions(+), 10 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index bc987da..7b65bdf 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -60,6 +60,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import javax.annotation.Nullable; import javax.management.NotCompliantMBeanException; +import javax.management.openmbean.CompositeData; import com.google.common.base.Function; import com.google.common.base.Predicates; @@ -107,6 +108,9 @@ import org.apache.jackrabbit.oak.spi.state.NodeStateDiff; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.jackrabbit.oak.stats.Clock; import org.apache.jackrabbit.oak.util.PerfLogger; +import org.apache.jackrabbit.stats.TimeSeriesMax; +import org.apache.jackrabbit.stats.TimeSeriesRecorder; +import org.apache.jackrabbit.stats.TimeSeriesStatsUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -369,6 +373,8 @@ public final class DocumentNodeStore private final DocumentNodeStoreMBean mbean; + private final BackgroundTaskExecutionStats backgroundTaskExecutionStats; + public DocumentNodeStore(DocumentMK.Builder builder) { this.blobStore = builder.getBlobStore(); if (builder.isUseSimpleRevision()) { @@ -459,13 +465,16 @@ public final class DocumentNodeStore return DocumentNodeStore.this.newRevision(); } }; + + backgroundTaskExecutionStats = new BackgroundTaskExecutionStats(); + batchCommitQueue = new BatchCommitQueue(store, revisionComparator); backgroundReadThread = new Thread( - new BackgroundReadOperation(this, isDisposed), + new BackgroundReadOperation(this, isDisposed, backgroundTaskExecutionStats), "DocumentNodeStore background read thread"); backgroundReadThread.setDaemon(true); backgroundUpdateThread = new Thread( - new BackgroundOperation(this, isDisposed), + new BackgroundOperation(this, isDisposed, backgroundTaskExecutionStats), "DocumentNodeStore background update thread"); backgroundUpdateThread.setDaemon(true); checkLastRevRecovery(); @@ -477,7 +486,7 @@ public final class DocumentNodeStore if (clusterNodeInfo != null) { leaseUpdateThread = new Thread( - new BackgroundLeaseUpdate(this, isDisposed), + new BackgroundLeaseUpdate(this, isDisposed, backgroundTaskExecutionStats), "DocumentNodeStore lease update thread"); leaseUpdateThread.setDaemon(true); leaseUpdateThread.start(); @@ -1648,6 +1657,7 @@ public final class DocumentNodeStore private synchronized void internalRunBackgroundReadOperations() { long start = clock.getTime(); // pull in changes from other cluster nodes + // OAK-2065 - background read BackgroundReadStats readStats = backgroundRead(true); long readTime = clock.getTime() - start; String msg = "Background read operations stats (read:{} {})"; @@ -2391,17 +2401,104 @@ public final class DocumentNodeStore } return sdf.format(r.getTimestamp()); } + + @Override + public CompositeData getBackgroundReadCounterTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.readCounter, "BackgroundReadCounter"); + } + + @Override + public CompositeData getBackgroundReadTimerTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.readTimer, "BackgroundReadTimer"); + } + + @Override + public CompositeData getBackgroundWriteCounterTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.writeCounter, "BackgroundWriteCounter"); + } + + @Override + public CompositeData getBackgroundWriteTimerTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.writeTimer, "BackgroundWriteTimer"); + } + + @Override + public CompositeData getBackgroundLeaseUpdateCounterTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.leaseUpdateCounter, "LeaseUpdateCounter"); + } + + @Override + public CompositeData getBackgroundLeaseUpdateTimerTimeSeries() { + return TimeSeriesStatsUtil.asCompositeData(backgroundTaskExecutionStats.leaseUpdateTimer, "LeaseUpdateTimer"); + } + + } + + BackgroundTaskExecutionStats getBackgroundTaskExecutionStats() { + return backgroundTaskExecutionStats; + } + + /** + * Records executions statistics related to the background operations performed by a DocumentNodeStore instance + * + *

The {@link #recordOneSecond()} method must be invoked once per second to make sure that the gathered statistics are consistent. The + * operation results are exposed through JMX

+ */ + class BackgroundTaskExecutionStats { + + private final TimeSeriesRecorder readCounter = new TimeSeriesRecorder(true); + private final TimeSeriesRecorder readTimer = new TimeSeriesRecorder(true); + + private final TimeSeriesRecorder writeCounter = new TimeSeriesRecorder(true); + private final TimeSeriesRecorder writeTimer = new TimeSeriesRecorder(true); + + private final TimeSeriesRecorder leaseUpdateCounter = new TimeSeriesRecorder(true); + private final TimeSeriesRecorder leaseUpdateTimer = new TimeSeriesRecorder(true); + + public void recordOperation(Class taskKind, long durationInMillis) { + TimeSeriesRecorder counter, timer; + if ( taskKind == BackgroundReadOperation.class ) { + counter = readCounter; + timer = readTimer; + } else if ( taskKind == BackgroundOperation.class) { + counter = writeCounter; + timer = writeTimer; + } else if ( taskKind == BackgroundLeaseUpdate.class) { + counter = leaseUpdateCounter; + timer = leaseUpdateTimer; + } else { + LOG.warn("No TimeSeriesRecorder defined for operation {}, execution stats will not be recorded"); + return; + } + + counter.getCounter().incrementAndGet(); + timer.getCounter().addAndGet(durationInMillis); + } + + public void recordOneSecond() { + + readCounter.recordOneSecond(); + readTimer.recordOneSecond(); + + writeCounter.recordOneSecond(); + writeTimer.recordOneSecond(); + + leaseUpdateCounter.recordOneSecond(); + leaseUpdateTimer.recordOneSecond(); + } } static abstract class NodeStoreTask implements Runnable { final WeakReference ref; private final AtomicBoolean isDisposed; private int delay; + private final BackgroundTaskExecutionStats executionStats; - NodeStoreTask(DocumentNodeStore nodeStore, AtomicBoolean isDisposed) { + NodeStoreTask(DocumentNodeStore nodeStore, AtomicBoolean isDisposed, BackgroundTaskExecutionStats executionStats) { ref = new WeakReference(nodeStore); delay = nodeStore.getAsyncDelay(); this.isDisposed = isDisposed; + this.executionStats = executionStats; } protected abstract void execute(@Nonnull DocumentNodeStore nodeStore); @@ -2418,8 +2515,10 @@ public final class DocumentNodeStore } DocumentNodeStore nodeStore = ref.get(); if (nodeStore != null) { + long start = nodeStore.getClock().getTime(); try { execute(nodeStore); + executionStats.recordOperation(getClass(), nodeStore.getClock().getTime() - start); } catch (Throwable t) { LOG.warn("Background operation failed: " + t.toString(), t); } @@ -2438,8 +2537,8 @@ public final class DocumentNodeStore static class BackgroundOperation extends NodeStoreTask { BackgroundOperation(DocumentNodeStore nodeStore, - AtomicBoolean isDisposed) { - super(nodeStore, isDisposed); + AtomicBoolean isDisposed, BackgroundTaskExecutionStats executionStats) { + super(nodeStore, isDisposed, executionStats); } @Override @@ -2454,8 +2553,8 @@ public final class DocumentNodeStore static class BackgroundReadOperation extends NodeStoreTask { BackgroundReadOperation(DocumentNodeStore nodeStore, - AtomicBoolean isDisposed) { - super(nodeStore, isDisposed); + AtomicBoolean isDisposed, BackgroundTaskExecutionStats executionStats) { + super(nodeStore, isDisposed, executionStats); } @Override @@ -2467,8 +2566,8 @@ public final class DocumentNodeStore static class BackgroundLeaseUpdate extends NodeStoreTask { BackgroundLeaseUpdate(DocumentNodeStore nodeStore, - AtomicBoolean isDisposed) { - super(nodeStore, isDisposed); + AtomicBoolean isDisposed, BackgroundTaskExecutionStats executionStats) { + super(nodeStore, isDisposed, executionStats); } @Override diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java index c3f9e3a..58056e5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreMBean.java @@ -19,6 +19,8 @@ package org.apache.jackrabbit.oak.plugins.document; +import javax.management.openmbean.CompositeData; + import org.apache.jackrabbit.oak.commons.jmx.Name; @SuppressWarnings("UnusedDeclaration") @@ -40,4 +42,16 @@ public interface DocumentNodeStoreMBean { String[] getLastKnownRevisions(); String formatRevision(@Name("revision") String rev, @Name("UTC")boolean utc); + + CompositeData getBackgroundReadTimerTimeSeries(); + + CompositeData getBackgroundReadCounterTimeSeries(); + + CompositeData getBackgroundWriteTimerTimeSeries(); + + CompositeData getBackgroundWriteCounterTimeSeries(); + + CompositeData getBackgroundLeaseUpdateCounterTimeSeries(); + + CompositeData getBackgroundLeaseUpdateTimerTimeSeries(); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java index defcd28..6797a69 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java @@ -410,6 +410,12 @@ public class DocumentNodeStoreService { registerJMXBeans(mk.getNodeStore()); registerLastRevRecoveryJob(mk.getNodeStore()); + registrations.add(WhiteboardUtils.scheduleWithFixedDelay(whiteboard, new Runnable() { + @Override + public void run() { + mk.getNodeStore().getBackgroundTaskExecutionStats().recordOneSecond(); + } + }, 1)); NodeStore store; DocumentNodeStore mns = mk.getNodeStore(); -- 2.4.1