Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java (revision 1846388) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java (working copy) @@ -17,11 +17,7 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import org.apache.jackrabbit.oak.api.CommitFailedException; -import org.apache.jackrabbit.oak.plugins.index.ContextAwareCallback; -import org.apache.jackrabbit.oak.plugins.index.IndexEditor; -import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; -import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; -import org.apache.jackrabbit.oak.plugins.index.IndexingContext; +import org.apache.jackrabbit.oak.plugins.index.*; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.ActiveDeletedBlobCollectorFactory.BlobDeletionCallback; @@ -35,8 +31,10 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyQuery; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig; +import org.apache.jackrabbit.oak.plugins.index.search.CompositePropertyUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.search.ExtractedTextCache; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.search.spi.editor.FulltextIndexWriterFactory; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; import org.apache.jackrabbit.oak.spi.commit.CommitContext; @@ -46,11 +44,15 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collection; +import java.util.LinkedList; + import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; @@ -75,6 +77,10 @@ private boolean nrtIndexingEnabled; private LuceneIndexWriterConfig writerConfig = new LuceneIndexWriterConfig(); + private final LuceneIndexMBean mbean; + private final StatisticsProvider statisticsProvider; + + /** * Number of indexed Lucene document that can be held in memory * This ensures that for very large commit memory consumption @@ -109,7 +115,7 @@ @Nullable IndexAugmentorFactory augmentorFactory, MountInfoProvider mountInfoProvider) { this(indexCopier, indexTracker, extractedTextCache, augmentorFactory, mountInfoProvider, - ActiveDeletedBlobCollectorFactory.NOOP); + ActiveDeletedBlobCollectorFactory.NOOP, null, null); } public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, @Nullable IndexTracker indexTracker, @@ -116,7 +122,9 @@ ExtractedTextCache extractedTextCache, @Nullable IndexAugmentorFactory augmentorFactory, MountInfoProvider mountInfoProvider, - @NotNull ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector) { + @NotNull ActiveDeletedBlobCollectorFactory.ActiveDeletedBlobCollector activeDeletedBlobCollector, + @Nullable LuceneIndexMBean mbean, + @Nullable StatisticsProvider statisticsProvider) { this.indexCopier = indexCopier; this.indexTracker = indexTracker; this.extractedTextCache = extractedTextCache != null ? extractedTextCache : new ExtractedTextCache(0, 0); @@ -123,6 +131,8 @@ this.augmentorFactory = augmentorFactory; this.mountInfoProvider = checkNotNull(mountInfoProvider); this.activeDeletedBlobCollector = activeDeletedBlobCollector; + this.mbean = mbean; + this.statisticsProvider = statisticsProvider; } @Override @@ -140,7 +150,8 @@ LuceneIndexDefinition indexDefinition = null; boolean asyncIndexing = true; String indexPath = indexingContext.getIndexPath(); - PropertyIndexUpdateCallback propertyUpdateCallback = null; + Collection callbacks = new LinkedList<>(); + PropertyIndexUpdateCallback propertyIndexUpdateCallback = null; if (nrtIndexingEnabled() && !indexingContext.isAsync() && IndexDefinition.supportsSyncOrNRTIndexing(definition)) { @@ -184,10 +195,10 @@ } if (indexDefinition.hasSyncPropertyDefinitions()) { - propertyUpdateCallback = new PropertyIndexUpdateCallback(indexPath, definition, root); + propertyIndexUpdateCallback = new PropertyIndexUpdateCallback(indexPath, definition, root); if (indexTracker != null) { PropertyQuery query = new LuceneIndexPropertyQuery(indexTracker, indexPath); - propertyUpdateCallback.getUniquenessConstraintValidator().setSecondStore(query); + propertyIndexUpdateCallback.getUniquenessConstraintValidator().setSecondStore(query); } } @@ -208,7 +219,17 @@ LuceneIndexEditorContext context = new LuceneIndexEditorContext(root, definition, indexDefinition, callback, writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing); - context.setPropertyUpdateCallback(propertyUpdateCallback); + if (propertyIndexUpdateCallback != null) { + callbacks.add(propertyIndexUpdateCallback); + } + if (mbean != null && statisticsProvider != null) { + callbacks.add(new LuceneIndexStatsUpdateCallback(indexPath, mbean, statisticsProvider)); + } + + if (!callbacks.isEmpty()) { + CompositePropertyUpdateCallback compositePropertyUpdateCallback = new CompositePropertyUpdateCallback(callbacks); + context.setPropertyUpdateCallback(compositePropertyUpdateCallback); + } return new LuceneIndexEditor(context); } return null; Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java (revision 1846388) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBean.java (working copy) @@ -129,4 +129,10 @@ @Description("Fetches hybrid property index info as json for index at given path") String getHybridIndexInfo(@Name("indexPath") String indexPath); + @Description("Fetches index size for index at given path") + String getSize(@Name("indexPath") String indexPath) throws IOException; + + @Description("Fetches current number of docs for index at given path") + String getDocCount(@Name("indexPath") String indexPath) throws IOException; + } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (revision 1846388) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (working copy) @@ -21,11 +21,8 @@ import java.io.File; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; import javax.management.openmbean.CompositeDataSupport; import javax.management.openmbean.CompositeType; @@ -60,7 +57,6 @@ import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; -import org.apache.jackrabbit.oak.plugins.index.search.IndexNode; import org.apache.jackrabbit.oak.plugins.index.search.util.NodeStateCloner; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; @@ -141,6 +137,21 @@ return tds; } + private IndexStats getIndexStats(String path) throws IOException { + LuceneIndexNode indexNode = null; + try { + indexNode = indexTracker.acquireIndexNode(path); + if (indexNode != null) { + return new IndexStats(path, indexNode); + } + } finally { + if (indexNode != null) { + indexNode.release(); + } + } + throw new IOException("could not fetch stats for index at path " + path); + } + @Override public TabularData getBadIndexStats() { TabularDataSupport tds; @@ -368,6 +379,16 @@ return new HybridPropertyIndexInfo(idx).getInfoAsJson(); } + @Override + public String getSize(String indexPath) throws IOException { + return String.valueOf(getIndexStats(indexPath).indexSize); + } + + @Override + public String getDocCount(String indexPath) throws IOException { + return String.valueOf(getIndexStats(indexPath).numDocs); + } + private Result getConsistencyCheckResult(String indexPath, boolean fullCheck) throws IOException { NodeState root = nodeStore.getRoot(); Level level = fullCheck ? Level.FULL : Level.BLOBS_ONLY; Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java (revision 1846388) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java (working copy) @@ -85,7 +85,9 @@ import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.MeterStats; import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.apache.jackrabbit.oak.stats.StatsOptions; import org.apache.lucene.analysis.util.CharFilterFactory; import org.apache.lucene.analysis.util.TokenFilterFactory; import org.apache.lucene.analysis.util.TokenizerFactory; @@ -382,17 +384,20 @@ regs.add(bundleContext.registerService(QueryIndexProvider.class.getName(), indexProvider, null)); registerObserver(bundleContext, config); registerLocalIndexObserver(bundleContext, tracker, config); - registerIndexEditor(bundleContext, tracker, config); + registerIndexInfoProvider(bundleContext); registerIndexImporterProvider(bundleContext); registerPropertyIndexCleaner(config, bundleContext); + LuceneIndexMBeanImpl mBean = new LuceneIndexMBeanImpl(tracker, nodeStore, indexPathService, getIndexCheckDir(), cleaner); oakRegs.add(registerMBean(whiteboard, LuceneIndexMBean.class, - new LuceneIndexMBeanImpl(indexProvider.getTracker(), nodeStore, indexPathService, getIndexCheckDir(), cleaner), + mBean, LuceneIndexMBean.TYPE, "Lucene Index statistics")); - registerGCMonitor(whiteboard, indexProvider.getTracker()); + registerGCMonitor(whiteboard, tracker); + + registerIndexEditor(bundleContext, tracker, mBean, config); } private File getIndexCheckDir() { @@ -491,16 +496,16 @@ } } - private void registerIndexEditor(BundleContext bundleContext, IndexTracker tracker, Map config) throws IOException { + private void registerIndexEditor(BundleContext bundleContext, IndexTracker tracker, LuceneIndexMBean mBean, Map config) throws IOException { boolean enableCopyOnWrite = PropertiesUtil.toBoolean(config.get(PROP_COPY_ON_WRITE), PROP_COPY_ON_WRITE_DEFAULT); if (enableCopyOnWrite){ initializeIndexCopier(bundleContext, config); editorProvider = new LuceneIndexEditorProvider(indexCopier, tracker, extractedTextCache, - augmentorFactory, mountInfoProvider, activeDeletedBlobCollector); + augmentorFactory, mountInfoProvider, activeDeletedBlobCollector, mBean, statisticsProvider); log.info("Enabling CopyOnWrite support. Index files would be copied under {}", indexDir.getAbsolutePath()); } else { editorProvider = new LuceneIndexEditorProvider(null, tracker, extractedTextCache, augmentorFactory, - mountInfoProvider, activeDeletedBlobCollector); + mountInfoProvider, activeDeletedBlobCollector, mBean, statisticsProvider); } editorProvider.setBlobStore(blobStore); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java (nonexistent) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java (working copy) @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.stats.CounterStats; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.apache.jackrabbit.oak.stats.StatsOptions; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; + +public class LuceneIndexStatsUpdateCallback implements PropertyUpdateCallback { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + private final String indexPath; + private final LuceneIndexMBean luceneIndexMBean; + private final StatisticsProvider statisticsProvider; + + LuceneIndexStatsUpdateCallback(String indexPath, @NotNull LuceneIndexMBean luceneIndexMBean, + @NotNull StatisticsProvider statisticsProvider) { + this.indexPath = indexPath; + this.luceneIndexMBean = luceneIndexMBean; + this.statisticsProvider = statisticsProvider; + } + + @Override + public void propertyUpdated(String nodePath, String propertyRelativePath, PropertyDefinition pd, @Nullable PropertyState before, @Nullable PropertyState after) { + // do nothing + } + + @Override + public void done() throws CommitFailedException { + try { + int docCount = Integer.parseInt(luceneIndexMBean.getDocCount(indexPath)); + CounterStats docCounter = statisticsProvider.getCounterStats(indexPath + "#no_docs", StatsOptions.METRICS_ONLY); + long previousCount = docCounter.getCount(); + long n = docCount - previousCount; + if (n > 0) { + docCounter.inc(n); + } else if (n < 0) { + docCounter.dec(n); + } + + long indexSize = Long.parseLong(luceneIndexMBean.getSize(indexPath)); + CounterStats sizeCounter = statisticsProvider.getCounterStats(indexPath + "#index_size", StatsOptions.METRICS_ONLY); + long previousSize = sizeCounter.getCount(); + long l = indexSize - previousSize; + if (l > 0) { + sizeCounter.inc(l); + } else if (l < 0) { + sizeCounter.dec(l); + } + + log.debug("{} stats updated; docCount {}, size {}", indexPath, docCount, indexSize); + } catch (IOException e) { + log.warn("could not update no_docs/index_size stats for index at {}", indexPath, e); + } + } +} Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ActiveDeletedBlobCollectorMBeanImplTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ActiveDeletedBlobCollectorMBeanImplTest.java (revision 1846388) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ActiveDeletedBlobCollectorMBeanImplTest.java (working copy) @@ -340,7 +340,7 @@ editorProvider = new LuceneIndexEditorProvider(null, null, new ExtractedTextCache(0, 0), null, Mounts.defaultMountInfoProvider(), - deletedFileTrackingADBC); + deletedFileTrackingADBC, null, null); repository = new Oak(dns2) .with(new OpenSecurityProvider()) .with(editorProvider) Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java (revision 1846388) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobCollectionIT.java (working copy) @@ -97,7 +97,7 @@ IndexCopier copier = createIndexCopier(); editorProvider = new LuceneIndexEditorProvider(copier, null, new ExtractedTextCache(10* FileUtils.ONE_MB,100), - null, Mounts.defaultMountInfoProvider(), adbc); + null, Mounts.defaultMountInfoProvider(), adbc, null, null); provider = new LuceneIndexProvider(copier); mongoConnection = connectionFactory.getConnection(); MongoUtils.dropCollections(mongoConnection.getDatabase()); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java (revision 1846388) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java (working copy) @@ -72,7 +72,7 @@ editorProvider = new LuceneIndexEditorProvider(copier, null, new ExtractedTextCache(10 * FileUtils.ONE_MB, 100), null, - Mounts.defaultMountInfoProvider(), adbc); + Mounts.defaultMountInfoProvider(), adbc, null, null); provider = new LuceneIndexProvider(copier); OakFileDataStore ds = new OakFileDataStore(); Index: oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/CompositePropertyUpdateCallback.java =================================================================== --- oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/CompositePropertyUpdateCallback.java (nonexistent) +++ oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/CompositePropertyUpdateCallback.java (working copy) @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.search; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.Collection; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Composite {@link PropertyUpdateCallback} + */ +public class CompositePropertyUpdateCallback implements PropertyUpdateCallback{ + + private final @NotNull Collection callbacks; + + public CompositePropertyUpdateCallback(@NotNull Collection callbacks) { + this.callbacks = callbacks; + checkNotNull(callbacks); + } + + @Override + public void propertyUpdated(String nodePath, String propertyRelativePath, PropertyDefinition pd, @Nullable PropertyState before, @Nullable PropertyState after) { + for (PropertyUpdateCallback callback : callbacks) { + callback.propertyUpdated(nodePath, propertyRelativePath, pd, before, after); + } + } + + @Override + public void done() throws CommitFailedException { + for (PropertyUpdateCallback callback : callbacks) { + callback.done(); + } + } +}