diff --git oak-lucene/pom.xml oak-lucene/pom.xml index 0ea1ccd..ce767aa 100644 --- oak-lucene/pom.xml +++ oak-lucene/pom.xml @@ -116,10 +116,14 @@ org.apache.lucene.sandbox.*;resolution:=optional, !org.apache.lucene.*, + !org.apache.jackrabbit.oak.cache, * - lucene-*;inline=true + lucene-*;inline=true, + + oak-core;inline="org/apache/jackrabbit/oak/cache/CacheStats*" + diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCache.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCache.java index f916d2b..760c2a0 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCache.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCache.java @@ -20,10 +20,16 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import java.io.IOException; +import java.util.concurrent.TimeUnit; import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.Weigher; import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.cache.CacheStats; import org.apache.jackrabbit.oak.commons.IOUtils; import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText; import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider; @@ -41,6 +47,24 @@ class ExtractedTextCache { private long totalTextSize; private long totalTime; private int preFetchedCount; + private final Cache cache; + private final CacheStats cacheStats; + + public ExtractedTextCache(long maxWeight, long expiryTimeInSecs) { + if (maxWeight > 0) { + cache = CacheBuilder.newBuilder() + .weigher(EmpiricalWeigher.INSTANCE) + .maximumWeight(maxWeight) + .expireAfterAccess(expiryTimeInSecs, TimeUnit.SECONDS) + .recordStats() + .build(); + cacheStats = new CacheStats(cache, "ExtractedTextCache", + EmpiricalWeigher.INSTANCE, maxWeight); + } else { + cache = null; + cacheStats = null; + } + } /** * Get the pre extracted text for given blob @@ -75,11 +99,21 @@ class ExtractedTextCache { log.warn("Error occurred while fetching pre extracted text for {}", propertyPath, e); } } + + String id = blob.getContentIdentity(); + if (cache != null && id != null && result == null) { + result = cache.getIfPresent(id); + } return result; } - public void put(Blob blob, ExtractedText extractedText){ - + public void put(@Nonnull Blob blob, @Nonnull ExtractedText extractedText) { + String id = blob.getContentIdentity(); + if (extractedText.getExtractionResult() == ExtractedText.ExtractionResult.SUCCESS + && cache != null + && id != null) { + cache.put(id, extractedText.getExtractedText().toString()); + } } public void addStats(int count, long timeInMillis, long bytesRead, long textLength){ @@ -123,6 +157,11 @@ class ExtractedTextCache { }; } + @CheckForNull + public CacheStats getCacheStats() { + return cacheStats; + } + public void setExtractedTextProvider(PreExtractedTextProvider extractedTextProvider) { this.extractedTextProvider = extractedTextProvider; } @@ -130,4 +169,31 @@ class ExtractedTextCache { public PreExtractedTextProvider getExtractedTextProvider() { return extractedTextProvider; } + + void resetCache(){ + if (cache != null){ + cache.invalidateAll(); + } + } + + //Taken from DocumentNodeStore and cache packages as they are private + private static class EmpiricalWeigher implements Weigher { + public static final EmpiricalWeigher INSTANCE = new EmpiricalWeigher(); + + private EmpiricalWeigher() { + } + + private static int getMemory(@Nonnull String s) { + return 16 // shallow size + + 40 + s.length() * 2; // value + } + + @Override + public int weigh(String key, String value) { + int size = 168; // overhead for each cache entry + size += getMemory(key); // key + size += getMemory(value); // value + return size; + } + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java index 08b05c6..b7ad24a 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java @@ -19,8 +19,6 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; - import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.plugins.index.IndexEditor; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; @@ -29,6 +27,8 @@ import org.apache.jackrabbit.oak.spi.commit.Editor; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; + /** * Service that provides Lucene based {@link IndexEditor}s * @@ -45,7 +45,7 @@ public class LuceneIndexEditorProvider implements IndexEditorProvider { } public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier) { - this(indexCopier, new ExtractedTextCache()); + this(indexCopier, new ExtractedTextCache(0, 0)); } public LuceneIndexEditorProvider(@Nullable IndexCopier indexCopier, diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index ed89c99..6292276 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -35,6 +35,7 @@ import javax.management.NotCompliantMBeanException; import com.google.common.base.Strings; import com.google.common.collect.Lists; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.felix.scr.annotations.Activate; import org.apache.felix.scr.annotations.Component; @@ -44,6 +45,8 @@ import org.apache.felix.scr.annotations.Reference; import org.apache.felix.scr.annotations.ReferenceCardinality; import org.apache.felix.scr.annotations.ReferencePolicy; import org.apache.felix.scr.annotations.ReferencePolicyOption; +import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; +import org.apache.jackrabbit.oak.cache.CacheStats; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; @@ -147,6 +150,23 @@ public class LuceneIndexProviderService { ) private static final String PROP_PREFETCH_INDEX_FILES = "prefetchIndexFiles"; + private static final int PROP_EXTRACTED_TEXT_CACHE_SIZE_DEFAULT = 0; + @Property( + intValue = PROP_EXTRACTED_TEXT_CACHE_SIZE_DEFAULT, + label = "Extracted text cache size (MB)", + description = "Cache size in MB for caching extracted text for some time. When set to 0 then " + + "cache would be disabled" + ) + private static final String PROP_EXTRACTED_TEXT_CACHE_SIZE = "extractedTextCacheSizeInMB"; + + private static final int PROP_EXTRACTED_TEXT_CACHE_EXPIRY_DEFAULT = 300; + @Property( + intValue = PROP_EXTRACTED_TEXT_CACHE_EXPIRY_DEFAULT, + label = "Extracted text cache expiry (secs)", + description = "Time in seconds for which the extracted text would be cached in memory" + ) + private static final String PROP_EXTRACTED_TEXT_CACHE_EXPIRY = "extractedTextCacheExpiryInSecs"; + private Whiteboard whiteboard; private BackgroundObserver backgroundObserver; @@ -168,7 +188,7 @@ public class LuceneIndexProviderService { private int threadPoolSize; - private ExtractedTextCache extractedTextCache = new ExtractedTextCache(); + private ExtractedTextCache extractedTextCache; @Activate private void activate(BundleContext bundleContext, Map config) @@ -183,7 +203,7 @@ public class LuceneIndexProviderService { initializeFactoryClassLoaders(getClass().getClassLoader()); whiteboard = new OsgiWhiteboard(bundleContext); threadPoolSize = PropertiesUtil.toInteger(config.get(PROP_THREAD_POOL_SIZE), PROP_THREAD_POOL_SIZE_DEFAULT); - + initializeExtractedTextCache(bundleContext, config); indexProvider = new LuceneIndexProvider(createTracker(bundleContext, config), scorerFactory); initializeLogging(config); initialize(); @@ -235,6 +255,10 @@ public class LuceneIndexProviderService { return indexCopier; } + ExtractedTextCache getExtractedTextCache() { + return extractedTextCache; + } + private void initialize(){ if(indexProvider == null){ return; @@ -394,6 +418,24 @@ public class LuceneIndexProviderService { TokenFilterFactory.reloadTokenFilters(classLoader); } + private void initializeExtractedTextCache(BundleContext bundleContext, Map config) { + int cacheSizeInMB = PropertiesUtil.toInteger(config.get(PROP_EXTRACTED_TEXT_CACHE_SIZE), + PROP_EXTRACTED_TEXT_CACHE_SIZE_DEFAULT); + int cacheExpiryInSecs = PropertiesUtil.toInteger(config.get(PROP_EXTRACTED_TEXT_CACHE_EXPIRY), + PROP_EXTRACTED_TEXT_CACHE_EXPIRY_DEFAULT); + + extractedTextCache = new ExtractedTextCache(cacheSizeInMB * FileUtils.ONE_MB, cacheExpiryInSecs); + + CacheStats stats = extractedTextCache.getCacheStats(); + if (stats != null){ + oakRegs.add(registerMBean(whiteboard, + CacheStatsMBean.class, stats, + CacheStatsMBean.TYPE, stats.getName())); + log.info("Extracted text caching enabled with maxSize {} MB, expiry time {} secs", + cacheSizeInMB, cacheExpiryInSecs); + } + } + private void registerExtractedTextProvider(PreExtractedTextProvider provider){ if (extractedTextCache != null){ if (provider != null){ diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCacheTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCacheTest.java new file mode 100644 index 0000000..12c2293 --- /dev/null +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/ExtractedTextCacheTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText; +import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; + +public class ExtractedTextCacheTest { + + @Test + public void cacheDisabling() throws Exception { + ExtractedTextCache cache = new ExtractedTextCache(0, 0); + assertNull(cache.getCacheStats()); + } + + @Test + public void cacheEnabled() throws Exception { + ExtractedTextCache cache = new ExtractedTextCache(10 * FileUtils.ONE_MB, 100); + assertNotNull(cache.getCacheStats()); + + Blob b = new IdBlob("hello", "a"); + String text = cache.get("/a", "foo", b, false); + assertNull(text); + + cache.put(b, new ExtractedText(ExtractedText.ExtractionResult.SUCCESS, "test hello")); + + text = cache.get("/a", "foo", b, false); + assertEquals("test hello", text); + } + + @Test + public void cacheEnabledNonIdBlob() throws Exception { + ExtractedTextCache cache = new ExtractedTextCache(10 * FileUtils.ONE_MB, 100); + + Blob b = new ArrayBasedBlob("hello".getBytes()); + String text = cache.get("/a", "foo", b, false); + assertNull(text); + + cache.put(b, new ExtractedText(ExtractedText.ExtractionResult.SUCCESS, "test hello")); + + text = cache.get("/a", "foo", b, false); + assertNull(text); + } + + @Test + public void cacheEnabledErrorInTextExtraction() throws Exception { + ExtractedTextCache cache = new ExtractedTextCache(10 * FileUtils.ONE_MB, 100); + + Blob b = new IdBlob("hello", "a"); + String text = cache.get("/a", "foo", b, false); + assertNull(text); + + cache.put(b, new ExtractedText(ExtractedText.ExtractionResult.ERROR, "test hello")); + + text = cache.get("/a", "foo", b, false); + assertNull(text); + } + + + private static class IdBlob extends ArrayBasedBlob { + final String id; + + public IdBlob(String value, String id) { + super(value.getBytes()); + this.id = id; + } + + @Override + public String getContentIdentity() { + return id; + } + } +} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java index 6263a3d..0146359 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java @@ -23,7 +23,9 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.Blob; +import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText; import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider; @@ -74,6 +76,7 @@ public class LuceneIndexProviderServiceTest { assertTrue(indexCopier.isPrefetchEnabled()); assertNotNull("CopyOnRead should be enabled by default", context.getService(CopyOnReadStatsMBean.class)); + assertNull(context.getService(CacheStatsMBean.class)); assertTrue(context.getService(Observer.class) instanceof BackgroundObserver); assertEquals(InfoStream.NO_OUTPUT, InfoStream.getDefault()); @@ -130,6 +133,23 @@ public class LuceneIndexProviderServiceTest { } @Test + public void enableExtractedTextCaching() throws Exception{ + Map config = getDefaultConfig(); + config.put("extractedTextCacheSizeInMB", 11); + MockOsgi.activate(service, context.bundleContext(), config); + + ExtractedTextCache textCache = service.getExtractedTextCache(); + assertNotNull(textCache.getCacheStats()); + assertNotNull(context.getService(CacheStatsMBean.class)); + + assertEquals(11 * FileUtils.ONE_MB, textCache.getCacheStats().getMaxTotalWeight()); + + MockOsgi.deactivate(service); + + assertNull(context.getService(CacheStatsMBean.class)); + } + + @Test public void preExtractedTextProvider() throws Exception{ MockOsgi.activate(service, context.bundleContext(), getDefaultConfig()); LuceneIndexEditorProvider editorProvider = diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index 8bc3093..a558564 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -41,6 +41,7 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.io.CountingInputStream; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.Oak; @@ -124,7 +125,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { @Override protected ContentRepository createRepository() { - editorProvider = new LuceneIndexEditorProvider(createIndexCopier()); + editorProvider = new LuceneIndexEditorProvider(createIndexCopier(), new ExtractedTextCache(10* FileUtils.ONE_MB, 100)); LuceneIndexProvider provider = new LuceneIndexProvider(); return new Oak() .with(new InitialContent()) @@ -1436,6 +1437,59 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { } @Test + public void preExtractedTextCache() throws Exception{ + Tree idx = createFulltextIndex(root.getTree("/"), "test"); + TestUtil.useV2(idx); + root.commit(); + + AccessStateProvidingBlob testBlob = + new AccessStateProvidingBlob("fox is jumping", "id1"); + + //1. Check by adding blobs in diff commit and reset + //cache each time. In such case blob stream would be + //accessed as many times + Tree test = root.getTree("/").addChild("test"); + createFileNode(test, "text", testBlob, "text/plain"); + root.commit(); + + editorProvider.getExtractedTextCache().resetCache(); + + test = root.getTree("/").addChild("test"); + createFileNode(test, "text2", testBlob, "text/plain"); + root.commit(); + + assertTrue(testBlob.isStreamAccessed()); + assertEquals(2, testBlob.accessCount); + + //Reset all test state + testBlob.resetState(); + editorProvider.getExtractedTextCache().resetCache(); + + //2. Now add 2 nodes with same blob in same commit + //This time cache effect would come and blob would + //be accessed only once + test = root.getTree("/").addChild("test"); + createFileNode(test, "text3", testBlob, "text/plain"); + createFileNode(test, "text4", testBlob, "text/plain"); + root.commit(); + + assertTrue(testBlob.isStreamAccessed()); + assertEquals(1, testBlob.accessCount); + + //Reset + testBlob.resetState(); + + //3. Now just add another node with same blob with no cache + //reset. This time blob stream would not be accessed at all + test = root.getTree("/").addChild("test"); + createFileNode(test, "text5", testBlob, "text/plain"); + root.commit(); + + assertFalse(testBlob.isStreamAccessed()); + assertEquals(0, testBlob.accessCount); + } + + @Test public void maxFieldLengthCheck() throws Exception{ Tree idx = createFulltextIndex(root.getTree("/"), "test"); TestUtil.useV2(idx); @@ -1879,6 +1933,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { private static class AccessStateProvidingBlob extends ArrayBasedBlob { private CountingInputStream stream; private String id; + private int accessCount; public AccessStateProvidingBlob(byte[] value) { super(value); @@ -1896,6 +1951,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { @Nonnull @Override public InputStream getNewStream() { + accessCount++; stream = new CountingInputStream(super.getNewStream()); return stream; } @@ -1906,6 +1962,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { public void resetState(){ stream = null; + accessCount = 0; } public long readByteCount(){