diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index c589260..8f20a29 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -52,6 +52,7 @@ import java.io.StringReader; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Deque; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -60,6 +61,9 @@ import java.util.concurrent.atomic.AtomicReference; import javax.jcr.PropertyType; +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Queues; +import com.google.common.collect.Sets; import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; @@ -163,9 +167,14 @@ public class LuceneIndex implements FulltextQueryIndex { private final NodeAggregator aggregator; - public LuceneIndex(Analyzer analyzer, NodeAggregator aggregator) { + private final SearcherManager searcherManager; + + public static final int LUCENE_QUERY_BATCH_SIZE = 100; + + public LuceneIndex(Analyzer analyzer, NodeAggregator aggregator, SearcherManager searcherManager) { this.analyzer = analyzer; this.aggregator = aggregator; + this.searcherManager = searcherManager; } @Override @@ -268,7 +277,7 @@ public class LuceneIndex implements FulltextQueryIndex { return false; } - private static Directory newDirectory(NodeState root) { + static Directory newDirectory(NodeState root) { NodeState def = getIndexDef(root); if (def == null) { return null; @@ -341,7 +350,7 @@ public class LuceneIndex implements FulltextQueryIndex { } @Override - public Cursor query(Filter filter, NodeState root) { + public Cursor query(final Filter filter, final NodeState root) { if (!isLive(root)) { throw new IllegalStateException("Lucene index is not live"); } @@ -350,77 +359,108 @@ public class LuceneIndex implements FulltextQueryIndex { if (relPaths.size() > 1) { return new MultiLuceneIndex(filter, root, relPaths).query(); } - String parent = relPaths.size() == 0 ? "" : relPaths.iterator().next(); + final String parent = relPaths.size() == 0 ? "" : relPaths.iterator().next(); + final int parentDepth = getDepth(parent); // we only restrict non-full-text conditions if there is // no relative property in the full-text constraint - boolean nonFullTextConstraints = parent.isEmpty(); - Directory directory = newDirectory(root); + final boolean nonFullTextConstraints = parent.isEmpty(); QueryEngineSettings settings = filter.getQueryEngineSettings(); - if (directory == null) { + if (!searcherManager.canAcquire(root)) { return newPathCursor(Collections. emptySet(), settings); } - long s = System.currentTimeMillis(); - try { - try { - IndexReader reader = DirectoryReader.open(directory); + Iterator itr = new AbstractIterator() { + private final Deque queue = Queues.newArrayDeque(); + private final Set seenPaths = Sets.newHashSet(); + private ScoreDoc lastDoc; + + @Override + protected LuceneResultRow computeNext() { + while (!queue.isEmpty() || loadDocs()) { + return queue.remove(); + } + return endOfData(); + } + + private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher) throws IOException { + String path = searcher.getIndexReader().document(doc.doc, + PATH_SELECTOR).get(PATH); + if (path != null) { + if ("".equals(path)) { + path = "/"; + } + if (!parent.isEmpty()) { + // TODO OAK-828 this breaks node aggregation + // get the base path + // ensure the path ends with the given + // relative path + // if (!path.endsWith("/" + parent)) { + // continue; + // } + path = getAncestorPath(path, parentDepth); + // avoid duplicate entries + if (seenPaths.contains(path)) { + return null; + } + seenPaths.add(path); + } + + LuceneResultRow r = new LuceneResultRow(); + r.path = path; + r.score = doc.score; + return r; + } + return null; + } + + /** + * Loads the lucene documents in batches + * @return true if any document is loaded + */ + private boolean loadDocs() { + IndexSearcher searcher = null; + ScoreDoc lastDocToRecord = null; try { - IndexSearcher searcher = new IndexSearcher(reader); - List rows = new ArrayList(); - Query query = getQuery(filter, reader, + searcher = acquire(); + Query query = getQuery(filter, searcher.getIndexReader(), nonFullTextConstraints, analyzer, getIndexDef(root)); + TopDocs docs = null; + if (lastDoc != null) { + docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE); + } else { + docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE); + } - // TODO OAK-828 - HashSet seenPaths = new HashSet(); - int parentDepth = getDepth(parent); - if (query != null) { - // OAK-925 - // TODO how to best avoid loading all entries in memory? - // (memory problem and performance problem) - TopDocs docs = searcher - .search(query, Integer.MAX_VALUE); - for (ScoreDoc doc : docs.scoreDocs) { - String path = reader.document(doc.doc, - PATH_SELECTOR).get(PATH); - if (path != null) { - if ("".equals(path)) { - path = "/"; - } - if (!parent.isEmpty()) { - // TODO OAK-828 this breaks node aggregation - // get the base path - // ensure the path ends with the given - // relative path - // if (!path.endsWith("/" + parent)) { - // continue; - // } - path = getAncestorPath(path, parentDepth); - // avoid duplicate entries - if (seenPaths.contains(path)) { - continue; - } - seenPaths.add(path); - } - - LuceneResultRow r = new LuceneResultRow(); - r.path = path; - r.score = doc.score; - rows.add(r); - } + for (ScoreDoc doc : docs.scoreDocs) { + LuceneResultRow row = convertToRow(doc, searcher); + if(row != null) { + queue.add(row); } + lastDocToRecord = doc; } - LOG.debug("query via {} took {} ms.", this, - System.currentTimeMillis() - s); - return new LucenePathCursor(rows, settings); + } catch (IOException e) { + LOG.warn("query via {} failed.", LuceneIndex.this, e); } finally { - reader.close(); + release(searcher); } - } finally { - directory.close(); + if (lastDocToRecord != null) { + this.lastDoc = lastDocToRecord; + } + return !queue.isEmpty(); } - } catch (IOException e) { - LOG.warn("query via {} failed.", this, e); - return newPathCursor(Collections. emptySet(), settings); - } + + private IndexSearcher acquire() throws IOException { + return searcherManager.acquire(root); + } + + private void release(IndexSearcher searcher) { + try { + searcherManager.release(searcher); + } catch (IOException e) { + LOG.warn("query via {} failed.", LuceneIndex.this, e); + } + } + }; + return new LucenePathCursor(itr, settings); } /** @@ -937,10 +977,7 @@ public class LuceneIndex implements FulltextQueryIndex { private final Cursor pathCursor; LuceneResultRow currentRow; - LucenePathCursor(List list, QueryEngineSettings settings) { - - final Iterator it = list.iterator(); - + LucenePathCursor(final Iterator it, QueryEngineSettings settings) { Iterator pathIterator = new Iterator() { @Override diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java index 1302635..1f14ba4 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java @@ -16,11 +16,16 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; +import java.io.Closeable; +import java.io.File; +import java.io.IOException; import java.util.List; +import java.util.concurrent.Executor; import javax.annotation.Nonnull; import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Deactivate; import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.spi.query.QueryIndex; @@ -29,6 +34,8 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.lucene.analysis.Analyzer; import com.google.common.collect.ImmutableList; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; /** * A provider for Lucene indexes. @@ -39,7 +46,7 @@ import com.google.common.collect.ImmutableList; @Component @Service(QueryIndexProvider.class) public class LuceneIndexProvider implements QueryIndexProvider, - LuceneIndexConstants { + LuceneIndexConstants, Closeable { /** * TODO how to inject this in an OSGi friendly way? @@ -48,6 +55,8 @@ public class LuceneIndexProvider implements QueryIndexProvider, protected NodeAggregator aggregator = null; + protected SearcherManager searcherManager = new SearcherManager(); + @Override @Nonnull public List getQueryIndexes(NodeState nodeState) { @@ -55,7 +64,7 @@ public class LuceneIndexProvider implements QueryIndexProvider, } protected LuceneIndex newLuceneIndex() { - return new LuceneIndex(analyzer, aggregator); + return new LuceneIndex(analyzer, aggregator, searcherManager); } /** @@ -72,6 +81,10 @@ public class LuceneIndexProvider implements QueryIndexProvider, this.aggregator = aggregator; } + public ReferenceManager getSearcherManager(){ + return searcherManager; + } + // ----- helper builder method public LuceneIndexProvider with(Analyzer analyzer) { @@ -84,4 +97,16 @@ public class LuceneIndexProvider implements QueryIndexProvider, return this; } + public LuceneIndexProvider with(File indexDir, Executor executor) { + this.searcherManager = new SearcherManager(indexDir,executor); + return this; + } + + @Deactivate + @Override + public void close() throws IOException { + if(searcherManager != null){ + searcherManager.close(); + } + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java new file mode 100644 index 0000000..aab2078 --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/SearcherManager.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.File; +import java.io.IOException; +import java.util.Set; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.CheckForNull; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.MoreExecutors; +import org.apache.commons.io.FileUtils; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.IOContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * Based on org.apache.lucene.search.SearcherManager but customised for + * Oak + */ +class SearcherManager extends ReferenceManager implements IndexReader.ReaderClosedListener { + private final Logger log = LoggerFactory.getLogger(getClass()); + private final File indexDir; + private Clock clock = Clock.SIMPLE; + + /** + * Default time interval after which any call made to acquire IndexSearcher + * would trigger a check for being current + */ + static final long RECHECK_INTERVAL = TimeUnit.SECONDS.toMillis(5); + private volatile NodeState root; + private final Executor executor; + private final Set filesToBeDeleted = Sets.newHashSet(); + private volatile long lastCheckTimeStamp; + + private volatile long lastVersion; + + public SearcherManager(){ + this(null, MoreExecutors.sameThreadExecutor()); + } + + /** + * If IndexDir is specified + * then the index content would be copied to the file system and that + * directory would be used + * + * @param indexDir dir where the index content should be copied to. If + * null then index would be read via OakDirectory + */ + public SearcherManager(File indexDir, Executor executor) { + this.indexDir = indexDir; + this.executor = executor; + } + + /** + * Checks if the IndexSearcher is initialized for given root or not + * If no OakDirectory can be created for given state then a IndexSearcher + * cannot be created + * @param state current state from which index information is to be read + * @return true if and only if an IndexSearcher is created + */ + public boolean canAcquire(NodeState state){ + if(current == null) { + synchronized (this) { + if(current == null) { + try { + current = createSearcher(getOakDirectory(state)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + } + return current != null; + } + + public IndexSearcher acquire(NodeState nodeState) throws IOException { + checkNotNull(current, "acquire should only be called if canAcquire returns true"); + if(recheckRequire()){ + checkForRefresh(nodeState); + } + return super.acquire(); + } + + @Override + protected void decRef(IndexSearcher reference) throws IOException { + reference.getIndexReader().decRef(); + //If refCount == 1 and reference != current indicates that reference + //is not in use and hence should be closed (ref starts with def count of 1) + if(reference.getIndexReader().getRefCount() == 1 + && current != reference){ + reference.getIndexReader().close(); + } + } + + @Override + protected IndexSearcher refreshIfNeeded(IndexSearcher referenceToRefresh) throws IOException { + final IndexReader r = referenceToRefresh.getIndexReader(); + assert r instanceof DirectoryReader: "searcher's IndexReader should be a DirectoryReader, but got " + r; + IndexSearcher newSearcher = null; + if(isOfNewerVersion(root)){ + newSearcher = createSearcher(getOakDirectory(root)); + } + return newSearcher; + } + + @Override + protected boolean tryIncRef(IndexSearcher reference) { + return reference.getIndexReader().tryIncRef(); + } + + @Override + protected int getRefCount(IndexSearcher reference) { + return reference.getIndexReader().getRefCount(); + } + + //~-------------------------------------------< ReaderCloseListener > + + @Override + public void onClose(IndexReader reader) { + if(reader instanceof DirectoryReader){ + try { + dispose(((DirectoryReader) reader).directory()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + //~--------------------------------------------< internal > + + + /** + * For testing purpose only + * + * @param clock custom clock to simulate time + */ + void setClock(Clock clock) { + this.clock = clock; + } + + private synchronized void checkForRefresh(NodeState nodeState) throws IOException { + lastCheckTimeStamp = clock.getTime(); + //Check if the Lucene dir from NodeState is of newer version + // - existing used directory + // - current stored root. If this was picked up in refreshIfNeeded by this time + // then it would be same as existing used directory. Otherwise if multiple + // checkForRefresh have been received before they got processed then it would + // be different and we would need to ensure that we keep the latest + if(isOfNewerVersion(nodeState) + && isOfNewerVersion(nodeState, root)){ + //Store the nodeState in root as we cannot pass argument to mayBeRefresh. So we + //save the state here and later when mayBeRefresh -> ifRefreshRequired is invoked + //the state is used for actual check + root = nodeState; + + //Do this in a sep thread as it might be expensive call + executor.execute(new Runnable() { + @Override + public void run() { + try { + SearcherManager.super.maybeRefresh(); + } catch (IOException e) { + log.warn("Error occurred while checking for refresh", e); + } + } + }); + } + } + + private boolean isOfNewerVersion(NodeState ns) throws IOException { + //TODO Here its assumed that segment versions are ascending + return getVersion(ns) > lastVersion; + } + + private boolean isOfNewerVersion(NodeState n1, NodeState n2) throws IOException { + return getVersion(n1) > getVersion(n2); + } + + private long getVersion(NodeState nodeState) throws IOException { + Directory dir = getOakDirectory(nodeState); + if(dir != null){ + SegmentInfos seg = new SegmentInfos(); + seg.read(dir); + return seg.getVersion(); + } + return -1; + } + + private boolean recheckRequire() { + return clock.getTime() - lastCheckTimeStamp > RECHECK_INTERVAL; + } + + + private IndexSearcher createSearcher(Directory oakDirectory) throws IOException { + if (oakDirectory == null) { + return null; + } + + Directory dirToUse = oakDirectory; + if (indexDir != null) { + if(!indexDir.exists()){ + FileUtils.forceMkdir(indexDir); + } + Directory fsdir = FSDirectory.open(indexDir); + copyDirectory(oakDirectory, fsdir); + dirToUse = fsdir; + } + + IndexReader reader = DirectoryReader.open(dirToUse); + reader.addReaderClosedListener(this); + lastVersion = ((DirectoryReader)reader).getVersion(); + return new IndexSearcher(reader); + } + + private void copyDirectory(Directory source, Directory dest) throws IOException { + //Following files would have to be copied always + //TODO Overwriting the segments.gen is probably safe when some Readers + //are still working with current directory as its content (latest segment) + //is already read by Reader + Set alwaysCopy = ImmutableSet.of("segments.gen"); + + //Files present in dest but not present in source have to be deleted + //Actual delete should happen when reader is getting disposed + Set deletedFiles = Sets.difference(ImmutableSet.copyOf(dest.listAll()), + ImmutableSet.copyOf(source.listAll())); + filesToBeDeleted.addAll(deletedFiles); + + for(String fileName : source.listAll()){ + if(!dest.fileExists(fileName) || alwaysCopy.contains(fileName)){ + source.copy(dest, fileName, fileName, IOContext.DEFAULT); + } + } + } + + @CheckForNull + private Directory getOakDirectory(NodeState root) { + return LuceneIndex.newDirectory(root); + } + + private void dispose(Directory directory) throws IOException { + //TODO Not sure if this is the best place to delete the files. + //As there may be in between readers. Probably a safe way would be + //to record this information in a file on file system and + //remove those files upon restart. As then no reader would possibly + //be accessing it + for(String file : filesToBeDeleted){ + directory.deleteFile(file); + } + log.debug("Following files have been removed from Lucene index directory [{}]", filesToBeDeleted); + filesToBeDeleted.clear(); + directory.close(); + } +} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java index f6a0342..af5815d 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java @@ -30,13 +30,13 @@ public class LowCostLuceneIndexProvider extends LuceneIndexProvider { @Override protected LuceneIndex newLuceneIndex() { - return new LowCostLuceneIndex(analyzer, aggregator); + return new LowCostLuceneIndex(analyzer, aggregator, searcherManager); } private static class LowCostLuceneIndex extends LuceneIndex { - public LowCostLuceneIndex(Analyzer analyzer, NodeAggregator aggregator) { - super(analyzer, aggregator); + public LowCostLuceneIndex(Analyzer analyzer, NodeAggregator aggregator, SearcherManager searcherManager) { + super(analyzer, aggregator, searcherManager); } @Override diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java index bd74f74..2485517 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java @@ -71,7 +71,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager()); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -98,7 +98,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager()); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -130,7 +130,7 @@ public class LuceneIndexTest { NodeState indexed = HOOK.processCommit(before, after,CommitInfo.EMPTY); - QueryIndex queryIndex = new LuceneIndex(analyzer, null); + QueryIndex queryIndex = new LuceneIndex(analyzer, null, new SearcherManager()); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java index da47633..6552ab8 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/BenchmarkRunner.java @@ -210,7 +210,8 @@ public class BenchmarkRunner { flatStructure.value(options)), new FullTextSearchTest( wikipedia.value(options), - report.value(options)) + report.value(options), + base.value(options)) }; Set argset = Sets.newHashSet(options.nonOptionArguments()); diff --git oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java index 30ca7f5..ffbc782 100644 --- oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java +++ oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/FullTextSearchTest.java @@ -23,6 +23,8 @@ import java.io.File; import java.util.List; import java.util.Random; import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import javax.jcr.Node; import javax.jcr.Repository; @@ -40,6 +42,7 @@ import com.google.common.base.CharMatcher; import com.google.common.base.Splitter; import com.google.common.collect.Lists; import com.google.common.collect.Sets; +import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.fixture.JcrCustomizer; import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture; import org.apache.jackrabbit.oak.fixture.RepositoryFixture; @@ -48,6 +51,9 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper; import org.apache.jackrabbit.util.Text; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.store.AlreadyClosedException; import static com.google.common.base.Preconditions.checkArgument; @@ -55,14 +61,22 @@ public class FullTextSearchTest extends AbstractTest sampleSet; private Random random; - private int maxRowsToFetch = Integer.getInteger("maxRowsToFetch",10000); + private final boolean copyIndexToDir = Boolean.getBoolean("copyIndexToDir"); + private final int maxRowsToFetch = Integer.getInteger("maxRowsToFetch",100); private TestContext defaultContext; + private final File indexDir; + private volatile int totalRead = 0; + private final ExecutorService executor = Executors.newSingleThreadExecutor(); - public FullTextSearchTest(File dump, boolean doReport) { + + public FullTextSearchTest(File dump, boolean doReport, File value) { this.dump = dump; this.doReport = doReport; + this.basedir = value; + this.indexDir = copyIndexToDir ? new File(basedir, "lucene"+System.currentTimeMillis()) : null; } @Override @@ -74,6 +88,13 @@ public class FullTextSearchTest extends AbstractTest