From 810af0916241c549984041f5358464dbfd66856c Mon Sep 17 00:00:00 2001 From: Vikas Saurabh Date: Thu, 24 Nov 2016 21:42:27 +0530 Subject: [PATCH] OAK-4726: Add Metrics coverage to the Lucene Indexer Refactored patch. Original patch as provided by Ian Boston. --- .../plugins/index/lucene/LuceneIndexEditor.java | 20 +++++++++++++++ .../index/lucene/LuceneIndexEditorContext.java | 10 +++++++- .../index/lucene/LuceneIndexEditorProvider.java | 17 ++++++++++-- .../index/lucene/LuceneIndexProviderService.java | 6 ++--- .../index/lucene/writer/DefaultIndexWriter.java | 30 ++++++++++++++++++++++ .../lucene/writer/DefaultIndexWriterFactory.java | 14 ++++++++-- .../lucene/writer/MultiplexingIndexWriter.java | 8 ++++-- 7 files changed, 95 insertions(+), 10 deletions(-) diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 21d28dc..b31c3ed 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -46,6 +46,9 @@ import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.plugins.memory.StringPropertyState; import org.apache.jackrabbit.oak.spi.commit.Editor; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.apache.jackrabbit.oak.stats.StatsOptions; +import org.apache.jackrabbit.oak.stats.TimerStats; import org.apache.jackrabbit.oak.util.BlobByteSource; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; @@ -86,6 +89,11 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { private final LuceneIndexEditorContext context; + private final StatisticsProvider statisticsProvider; + + private final String cachedTextExtractionStatsName; + private final String textExtractionStatsName; + /** Name of this node, or {@code null} for the root node. */ private final String name; @@ -117,6 +125,9 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { this.name = null; this.path = "/"; this.context = context; + this.statisticsProvider = context.getStatisticsProvider(); + this.cachedTextExtractionStatsName = getClass().getName() + "-cachedTextExtraction"; + this.textExtractionStatsName = getClass().getName() + "-textExtraction"; this.isDeleted = false; this.matcherState = MatcherState.NONE; this.pathFilterResult = context.getDefinition().getPathFilter().filter(PathUtils.ROOT_PATH); @@ -130,6 +141,9 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { this.name = name; this.path = null; this.context = parent.context; + this.statisticsProvider = parent.statisticsProvider; + this.cachedTextExtractionStatsName = getClass().getName() + "-cachedTextExtraction"; + this.textExtractionStatsName = getClass().getName() + "-textExtraction"; this.isDeleted = isDeleted; this.matcherState = matcherState; this.pathFilterResult = pathFilterResult; @@ -950,9 +964,15 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { //Skip text extraction for sync indexing return null; } + TimerStats.Context cachedTimer = + statisticsProvider.getTimer(cachedTextExtractionStatsName, StatsOptions.METRICS_ONLY).time(); String text = context.getExtractedTextCache().get(path, propertyName, v, context.isReindex()); + cachedTimer.stop(); if (text == null){ + TimerStats.Context extractionTimer = + statisticsProvider.getTimer(textExtractionStatsName, StatsOptions.METRICS_ONLY).time(); text = parseStringValue0(v, metadata, path); + extractionTimer.stop(); } return text; } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 745889f..b2442a6 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -36,6 +36,7 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFa import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.jackrabbit.oak.util.PerfLogger; import org.apache.jackrabbit.util.ISO8601; import org.apache.lucene.facet.FacetsConfig; @@ -97,6 +98,7 @@ public class LuceneIndexEditorContext { //Intentionally static, so that it can be set without passing around clock objects //Set for testing ONLY private static Clock clock = Clock.SIMPLE; + private StatisticsProvider statisticsProvider; LuceneIndexEditorContext(NodeState root, NodeBuilder definition, @Nullable IndexDefinition indexDefinition, @@ -104,7 +106,8 @@ public class LuceneIndexEditorContext { LuceneIndexWriterFactory indexWriterFactory, ExtractedTextCache extractedTextCache, IndexAugmentorFactory augmentorFactory, - IndexingContext indexingContext, boolean asyncIndexing) { + IndexingContext indexingContext, boolean asyncIndexing, + StatisticsProvider statisticsProvider) { configureUniqueId(definition); this.root = root; this.indexingContext = checkNotNull(indexingContext); @@ -119,6 +122,7 @@ public class LuceneIndexEditorContext { if (this.definition.isOfOldFormat()){ IndexDefinition.updateDefinition(definition); } + this.statisticsProvider = statisticsProvider; } Parser getParser() { @@ -309,6 +313,10 @@ public class LuceneIndexEditorContext { return TikaConfig.getDefaultConfig(); } + public StatisticsProvider getStatisticsProvider() { + return statisticsProvider; + } + static class TextExtractionStats { /** * Log stats only if time spent is more than 2 min diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java index d771778..49192ab 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java @@ -36,6 +36,7 @@ import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -53,6 +54,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstant public class LuceneIndexEditorProvider implements IndexEditorProvider { private final Logger log = LoggerFactory.getLogger(getClass()); private final IndexCopier indexCopier; + private final StatisticsProvider statisticsProvider; private final ExtractedTextCache extractedTextCache; private final IndexAugmentorFactory augmentorFactory; private final LuceneIndexWriterFactory indexWriterFactory; @@ -91,11 +93,22 @@ public class LuceneIndexEditorProvider implements IndexEditorProvider { ExtractedTextCache extractedTextCache, @Nullable IndexAugmentorFactory augmentorFactory, MountInfoProvider mountInfoProvider) { + this(indexCopier, indexTracker, extractedTextCache, augmentorFactory, mountInfoProvider, StatisticsProvider.NOOP); + + } + + public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, + @Nullable IndexTracker indexTracker, + ExtractedTextCache extractedTextCache, + @Nullable IndexAugmentorFactory augmentorFactory, + MountInfoProvider mountInfoProvider, + StatisticsProvider statisticsProvider) { this.indexCopier = indexCopier; + this.statisticsProvider = statisticsProvider; this.indexTracker = indexTracker; this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0); this.augmentorFactory = augmentorFactory; - this.indexWriterFactory = new DefaultIndexWriterFactory(checkNotNull(mountInfoProvider), indexCopier); + this.indexWriterFactory = new DefaultIndexWriterFactory(checkNotNull(mountInfoProvider), indexCopier, statisticsProvider); } @Override @@ -151,7 +164,7 @@ public class LuceneIndexEditorProvider implements IndexEditorProvider { } LuceneIndexEditorContext context = new LuceneIndexEditorContext(root, definition, indexDefinition, callback, - writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing); + writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing, statisticsProvider); return new LuceneIndexEditor(context); } return null; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index eea9477..7b6f724 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -360,11 +360,11 @@ public class LuceneIndexProviderService { if (enableCopyOnWrite){ initializeIndexCopier(bundleContext, config); editorProvider = new LuceneIndexEditorProvider(indexCopier, tracker, extractedTextCache, - augmentorFactory, mountInfoProvider); + augmentorFactory, mountInfoProvider, statisticsProvider); log.info("Enabling CopyOnWrite support. Index files would be copied under {}", indexDir.getAbsolutePath()); } else { - editorProvider = new LuceneIndexEditorProvider(null, tracker, extractedTextCache, augmentorFactory, - mountInfoProvider); + editorProvider = new LuceneIndexEditorProvider(null, tracker, + extractedTextCache, augmentorFactory, mountInfoProvider, statisticsProvider); } regs.add(bundleContext.registerService(IndexEditorProvider.class.getName(), editorProvider, null)); oakRegs.add(registerMBean(whiteboard, diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java index 4b47863..96cbcf8 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriter.java @@ -32,6 +32,10 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.stats.MeterStats; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.apache.jackrabbit.oak.stats.StatsOptions; +import org.apache.jackrabbit.oak.stats.TimerStats; import org.apache.jackrabbit.oak.util.PerfLogger; import org.apache.jackrabbit.util.ISO8601; import org.apache.lucene.analysis.Analyzer; @@ -61,36 +65,55 @@ class DefaultIndexWriter implements LuceneIndexWriter { private final String dirName; private final String suggestDirName; private final boolean reindex; + private final StatisticsProvider statisticsProvider; + private final String statsName; private IndexWriter writer; private Directory directory; public DefaultIndexWriter(IndexDefinition definition, NodeBuilder definitionBuilder, @Nullable IndexCopier indexCopier, String dirName, String suggestDirName, boolean reindex){ + this(definition, definitionBuilder, indexCopier, dirName, suggestDirName, reindex, StatisticsProvider.NOOP); + } + + public DefaultIndexWriter(IndexDefinition definition, NodeBuilder definitionBuilder, + @Nullable IndexCopier indexCopier, String dirName, String suggestDirName, boolean reindex, + StatisticsProvider statisticsProvider){ this.definition = definition; this.definitionBuilder = definitionBuilder; this.indexCopier = indexCopier; this.dirName = dirName; this.suggestDirName = suggestDirName; this.reindex = reindex; + this.statisticsProvider = statisticsProvider; + this.statsName = DefaultIndexWriter.class.getName()+"-"+dirName+"-"; } @Override public void updateDocument(String path, Iterable doc) throws IOException { + TimerStats.Context context = getContext("udpateDocument"); getWriter().updateDocument(newPathTerm(path), doc); + context.stop(); } + @Override public void deleteDocuments(String path) throws IOException { + TimerStats.Context context = getContext("deleteDocuments"); getWriter().deleteDocuments(newPathTerm(path)); getWriter().deleteDocuments(new PrefixQuery(newPathTerm(path + "/"))); + context.stop(); } void deleteAll() throws IOException { + TimerStats.Context context = getContext("deleteAll"); getWriter().deleteAll(); + context.stop(); } @Override public boolean close(long timestamp) throws IOException { + TimerStats.Context context = getContext("close"); + //If reindex or fresh index and write is null on close //it indicates that the index is empty. In such a case trigger //creation of write such that an empty Lucene index state is persisted @@ -127,6 +150,7 @@ class DefaultIndexWriter implements LuceneIndexWriter { directory.close(); PERF_LOGGER.end(start, -1, "Closed directory for directory {}", definition); } + context.stop(); return indexUpdated; } @@ -244,4 +268,10 @@ class DefaultIndexWriter implements LuceneIndexWriter { org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount(overallSize), sb.toString()); } + + + private TimerStats.Context getContext(String name) { + return statisticsProvider.getTimer(this.statsName + name, StatsOptions.METRICS_ONLY).time(); + } + } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java index 90cbc96..e08e57f 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/DefaultIndexWriterFactory.java @@ -26,23 +26,33 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants; import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; public class DefaultIndexWriterFactory implements LuceneIndexWriterFactory { private final MountInfoProvider mountInfoProvider; private final IndexCopier indexCopier; + private StatisticsProvider statisticsProvider; public DefaultIndexWriterFactory(MountInfoProvider mountInfoProvider, @Nullable IndexCopier indexCopier) { + this(mountInfoProvider, indexCopier, StatisticsProvider.NOOP); + } + + public DefaultIndexWriterFactory(MountInfoProvider mountInfoProvider, @Nullable IndexCopier indexCopier, + StatisticsProvider statisticsProvider) { this.mountInfoProvider = mountInfoProvider; this.indexCopier = indexCopier; + this.statisticsProvider = statisticsProvider; } @Override public LuceneIndexWriter newInstance(IndexDefinition definition, NodeBuilder definitionBuilder, boolean reindex) { if (mountInfoProvider.hasNonDefaultMounts()){ - return new MultiplexingIndexWriter(indexCopier, mountInfoProvider, definition, definitionBuilder, reindex); + return new MultiplexingIndexWriter(indexCopier, mountInfoProvider, definition, definitionBuilder, reindex, + statisticsProvider); } return new DefaultIndexWriter(definition, definitionBuilder, indexCopier, - LuceneIndexConstants.INDEX_DATA_CHILD_NAME, LuceneIndexConstants.SUGGEST_DATA_CHILD_NAME, reindex); + LuceneIndexConstants.INDEX_DATA_CHILD_NAME, LuceneIndexConstants.SUGGEST_DATA_CHILD_NAME, reindex, + statisticsProvider); } } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java index 6a76027..dacab95 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java @@ -28,22 +28,26 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; import org.apache.jackrabbit.oak.spi.mount.Mount; import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.lucene.index.IndexableField; class MultiplexingIndexWriter implements LuceneIndexWriter { private final IndexCopier indexCopier; private final MountInfoProvider mountInfoProvider; private final IndexDefinition definition; + private StatisticsProvider statisticsProvider; private final NodeBuilder definitionBuilder; private final boolean reindex; private final Map writers = Maps.newHashMap(); public MultiplexingIndexWriter(IndexCopier indexCopier, MountInfoProvider mountInfoProvider, - IndexDefinition definition, NodeBuilder definitionBuilder, boolean reindex) { + IndexDefinition definition, NodeBuilder definitionBuilder, boolean reindex, + StatisticsProvider statisticsProvider) { this.indexCopier = indexCopier; this.mountInfoProvider = mountInfoProvider; this.definition = definition; + this.statisticsProvider = statisticsProvider; this.definitionBuilder = definitionBuilder; this.reindex = reindex; } @@ -94,6 +98,6 @@ class MultiplexingIndexWriter implements LuceneIndexWriter { private DefaultIndexWriter createWriter(Mount m) { String dirName = MultiplexersLucene.getIndexDirName(m); String suggestDirName = MultiplexersLucene.getSuggestDirName(m); - return new DefaultIndexWriter(definition, definitionBuilder, indexCopier, dirName, suggestDirName, reindex); + return new DefaultIndexWriter(definition, definitionBuilder, indexCopier, dirName, suggestDirName, reindex, statisticsProvider); } } -- 2.8.3