diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java index 92cf846..fc2d9c3 100644 --- oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java +++ oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java @@ -491,6 +491,11 @@ public class Oak { return this.whiteboard; } + @Nonnull + public NodeStore getStore(){ + return store; + } + public ContentRepository createContentRepository() { whiteboard.register(Executor.class, executor, Collections.emptyMap()); diff --git oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java index 785e0c8..8c15f00 100644 --- oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java +++ oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java @@ -183,6 +183,10 @@ public class Jcr { return this; } + public Oak getOak() { + return oak; + } + public Repository createRepository() { return new RepositoryImpl( oak.createContentRepository(), diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index c589260..ae45df3 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -85,7 +85,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; @@ -163,9 +162,14 @@ public class LuceneIndex implements FulltextQueryIndex { private final NodeAggregator aggregator; - public LuceneIndex(Analyzer analyzer, NodeAggregator aggregator) { + private final SearcherManager searcherManager; + + public LuceneIndex(Analyzer analyzer, NodeAggregator aggregator, SearcherManager searcherManager) { this.analyzer = analyzer; this.aggregator = aggregator; + this.searcherManager = searcherManager; + //TODO Probably not required if we do the initialization in a sep job + this.searcherManager.initializeIfRequired(); } @Override @@ -268,7 +272,7 @@ public class LuceneIndex implements FulltextQueryIndex { return false; } - private static Directory newDirectory(NodeState root) { + static Directory newDirectory(NodeState root) { NodeState def = getIndexDef(root); if (def == null) { return null; @@ -354,68 +358,63 @@ public class LuceneIndex implements FulltextQueryIndex { // we only restrict non-full-text conditions if there is // no relative property in the full-text constraint boolean nonFullTextConstraints = parent.isEmpty(); - Directory directory = newDirectory(root); QueryEngineSettings settings = filter.getQueryEngineSettings(); - if (directory == null) { + if (!searcherManager.isInitialized()) { return newPathCursor(Collections. emptySet(), settings); } long s = System.currentTimeMillis(); try { + IndexSearcher searcher = searcherManager.acquire(); try { - IndexReader reader = DirectoryReader.open(directory); - try { - IndexSearcher searcher = new IndexSearcher(reader); - List rows = new ArrayList(); - Query query = getQuery(filter, reader, - nonFullTextConstraints, analyzer, getIndexDef(root)); - - // TODO OAK-828 - HashSet seenPaths = new HashSet(); - int parentDepth = getDepth(parent); - if (query != null) { - // OAK-925 - // TODO how to best avoid loading all entries in memory? - // (memory problem and performance problem) - TopDocs docs = searcher - .search(query, Integer.MAX_VALUE); - for (ScoreDoc doc : docs.scoreDocs) { - String path = reader.document(doc.doc, - PATH_SELECTOR).get(PATH); - if (path != null) { - if ("".equals(path)) { - path = "/"; - } - if (!parent.isEmpty()) { - // TODO OAK-828 this breaks node aggregation - // get the base path - // ensure the path ends with the given - // relative path - // if (!path.endsWith("/" + parent)) { - // continue; - // } - path = getAncestorPath(path, parentDepth); - // avoid duplicate entries - if (seenPaths.contains(path)) { - continue; - } - seenPaths.add(path); + IndexReader reader = searcher.getIndexReader(); + List rows = new ArrayList(); + Query query = getQuery(filter, reader, + nonFullTextConstraints, analyzer, getIndexDef(root)); + + // TODO OAK-828 + HashSet seenPaths = new HashSet(); + int parentDepth = getDepth(parent); + if (query != null) { + // OAK-925 + // TODO how to best avoid loading all entries in memory? + // (memory problem and performance problem) + TopDocs docs = searcher + .search(query, Integer.MAX_VALUE); + for (ScoreDoc doc : docs.scoreDocs) { + String path = reader.document(doc.doc, + PATH_SELECTOR).get(PATH); + if (path != null) { + if ("".equals(path)) { + path = "/"; + } + if (!parent.isEmpty()) { + // TODO OAK-828 this breaks node aggregation + // get the base path + // ensure the path ends with the given + // relative path + // if (!path.endsWith("/" + parent)) { + // continue; + // } + path = getAncestorPath(path, parentDepth); + // avoid duplicate entries + if (seenPaths.contains(path)) { + continue; } - - LuceneResultRow r = new LuceneResultRow(); - r.path = path; - r.score = doc.score; - rows.add(r); + seenPaths.add(path); } + + LuceneResultRow r = new LuceneResultRow(); + r.path = path; + r.score = doc.score; + rows.add(r); } } - LOG.debug("query via {} took {} ms.", this, - System.currentTimeMillis() - s); - return new LucenePathCursor(rows, settings); - } finally { - reader.close(); } + LOG.debug("query via {} took {} ms.", this, + System.currentTimeMillis() - s); + return new LucenePathCursor(rows, settings); } finally { - directory.close(); + searcherManager.release(searcher); } } catch (IOException e) { LOG.warn("query via {} failed.", this, e); diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java index 1302635..ad2a1ba 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java @@ -16,19 +16,25 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; +import java.io.Closeable; +import java.io.IOException; import java.util.List; import javax.annotation.Nonnull; import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Deactivate; import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.lucene.analysis.Analyzer; import com.google.common.collect.ImmutableList; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; /** * A provider for Lucene indexes. @@ -39,7 +45,7 @@ import com.google.common.collect.ImmutableList; @Component @Service(QueryIndexProvider.class) public class LuceneIndexProvider implements QueryIndexProvider, - LuceneIndexConstants { + LuceneIndexConstants, Closeable { /** * TODO how to inject this in an OSGi friendly way? @@ -48,6 +54,10 @@ public class LuceneIndexProvider implements QueryIndexProvider, protected NodeAggregator aggregator = null; + //TODO need to register a job with whiteboard to periodically call + //may be referesh + protected SearcherManager searcherManager; + @Override @Nonnull public List getQueryIndexes(NodeState nodeState) { @@ -55,7 +65,7 @@ public class LuceneIndexProvider implements QueryIndexProvider, } protected LuceneIndex newLuceneIndex() { - return new LuceneIndex(analyzer, aggregator); + return new LuceneIndex(analyzer, aggregator, searcherManager); } /** @@ -72,6 +82,14 @@ public class LuceneIndexProvider implements QueryIndexProvider, this.aggregator = aggregator; } + public void setNodeStore(NodeStore nodeStore) { + this.searcherManager = new SearcherManager(nodeStore); + } + + public ReferenceManager getSearcherManager(){ + return searcherManager; + } + // ----- helper builder method public LuceneIndexProvider with(Analyzer analyzer) { @@ -84,4 +102,20 @@ public class LuceneIndexProvider implements QueryIndexProvider, return this; } + public LuceneIndexProvider with(NodeStore nodeStore) { + this.setNodeStore(nodeStore); + return this; + } + + @Deactivate + private void deactivate() throws IOException { + close(); + } + + @Override + public void close() throws IOException { + if(searcherManager != null){ + searcherManager.close(); + } + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java new file mode 100644 index 0000000..5457bff --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; + +import javax.annotation.CheckForNull; + +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.store.Directory; + +/** + * Based on org.apache.lucene.search.SearcherManager but customised for + * Oak + */ +public class SearcherManager extends ReferenceManager { + private final NodeStore nodeStore; + private final NodeState root; + + public SearcherManager(NodeStore nodeStore) { + this.nodeStore = nodeStore; + this.root = null; + initializeIfRequired(); + } + + public SearcherManager(NodeState root) { + this.nodeStore = null; + this.root = root; + initializeIfRequired(); + } + + public boolean isInitialized(){ + return current != null; + } + + public boolean initializeIfRequired(){ + if(current == null){ + try { + current = getSearcher(); + } catch (IOException e) { + throw new RuntimeException(e); + } + return true; + } + return false; + } + + @Override + protected void decRef(IndexSearcher reference) throws IOException { + reference.getIndexReader().decRef(); + } + + @Override + protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException { + final IndexReader r = referenceToRefresh.getIndexReader(); + assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r; + if (isCurrent(r)) { + return null; + } else { + return getSearcher(); + } + } + + private boolean isCurrent(IndexReader r) throws IOException { + SegmentInfos sis = new SegmentInfos(); + sis.read(getDirectory()); + return sis.getVersion() == ((DirectoryReader)r).getVersion(); + } + + @Override + protected boolean tryIncRef(IndexSearcher reference) { + return reference.getIndexReader().tryIncRef(); + } + + @Override + protected int getRefCount(IndexSearcher reference) { + return reference.getIndexReader().getRefCount(); + } + + private IndexSearcher getSearcher() throws IOException { + Directory directory = getDirectory(); + if(directory == null){ + return null; + } + IndexReader reader = DirectoryReader.open(directory); + IndexSearcher searcher = new IndexSearcher(reader); + return searcher; + } + + @CheckForNull + private Directory getDirectory() { + NodeState root = nodeStore != null ? nodeStore.getRoot() : this.root; + return LuceneIndex.newDirectory(root); + } +} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java index f6a0342..af5815d 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java @@ -30,13 +30,13 @@ public class LowCostLuceneIndexProvider extends LuceneIndexProvider { @Override protected LuceneIndex newLuceneIndex() { - return new LowCostLuceneIndex(analyzer, aggregator); + return new LowCostLuceneIndex(analyzer, aggregator, searcherManager); } private static class LowCostLuceneIndex extends LuceneIndex { - public LowCostLuceneIndex(Analyzer analyzer, NodeAggregator aggregator) { - super(analyzer, aggregator); + public LowCostLuceneIndex(Analyzer analyzer, NodeAggregator aggregator, SearcherManager searcherManager) { + super(analyzer, aggregator, searcherManager); } @Override diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java index bd74f74..3114c95 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java @@ -71,7 +71,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager(indexed)); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -98,7 +98,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager(indexed)); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -130,7 +130,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after,CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager(indexed)); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java index 25936b9..35503fd 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java @@ -49,6 +49,9 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper; import org.apache.jackrabbit.util.Text; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.store.AlreadyClosedException; import static com.google.common.base.Preconditions.checkArgument; @@ -59,6 +62,7 @@ public class FullTextSearchTest extends AbstractTest { private List sampleSet; private Random random; private Reader reader; + private ReferenceManager searcherManager; public FullTextSearchTest(File dump, boolean doReport) { this.dump = dump; @@ -70,6 +74,14 @@ public class FullTextSearchTest extends AbstractTest { random = new Random(42); //fixed seed Session importSession = loginWriter(); sampleSet = importWikipedia(importSession); + if(searcherManager != null){ + try { + searcherManager.maybeRefresh(); + } catch (AlreadyClosedException ignore) { + //searcherManager yet not initialized. Would + //be initialize with first call + } + } reader = new Reader(); } @@ -100,7 +112,9 @@ public class FullTextSearchTest extends AbstractTest { return ((OakRepositoryFixture) fixture).setUpCluster(1, new JcrCustomizer() { @Override public Jcr customize(Jcr jcr) { - jcr.with(new LuceneIndexProvider()) + LuceneIndexProvider lip = new LuceneIndexProvider().with(jcr.getOak().getStore()); + searcherManager = lip.getSearcherManager(); + jcr.with(lip) .with(new LuceneIndexEditorProvider()) .with(new LuceneInitializerHelper("luceneGlobal", null)); return jcr;