Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java (revision 1851902) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java (working copy) @@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.EqualsDiff; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -195,39 +196,29 @@ refresh = true; } + /** + * Acquire the index node, if the index is good. + * + * @param path the index path + * @return the index node, or null if it's a bad (corrupt) index + */ + @Nullable public LuceneIndexNode acquireIndexNode(String path) { LuceneIndexNodeManager index = indices.get(path); LuceneIndexNode indexNode = index != null ? index.acquire() : null; if (indexNode != null) { return indexNode; - } else { - return findIndexNode(path); } + return findIndexNode(path); } + /** + * Get the index node, if the index is good. + * + * @param path the index path + * @return the index node, or null if it's a bad (corrupt) index + */ @Nullable - public LuceneIndexDefinition getIndexDefinition(String indexPath){ - LuceneIndexNodeManager node = indices.get(indexPath); - if (node != null){ - //Accessing the definition should not require - //locking as its immutable state - return node.getDefinition(); - } - return null; - } - - public Set getIndexNodePaths(){ - return indices.keySet(); - } - - public BadIndexTracker getBadIndexTracker() { - return badIndexTracker; - } - - public NodeState getRoot() { - return root; - } - private synchronized LuceneIndexNode findIndexNode(String path) { // Retry the lookup from acquireIndexNode now that we're // synchronized. The acquire() call is guaranteed to succeed @@ -270,6 +261,42 @@ return null; } + @Nullable + public LuceneIndexDefinition getIndexDefinition(String indexPath){ + LuceneIndexNodeManager indexNodeManager = indices.get(indexPath); + if (indexNodeManager != null) { + // Accessing the definition should not require + // locking as its immutable state + return indexNodeManager.getDefinition(); + } + // fallback - create definition from scratch + NodeState node = NodeStateUtils.getNode(root, indexPath); + if (!node.exists()) { + return null; + } + // only if there exists a stored index definition + if (!node.hasChildNode(INDEX_DEFINITION_NODE)) { + return null; + } + if (!isLuceneIndexNode(node)) { + return null; + } + // this will internally use the stored index definition + return new LuceneIndexDefinition(root, node, indexPath); + } + + public Set getIndexNodePaths(){ + return indices.keySet(); + } + + public BadIndexTracker getBadIndexTracker() { + return badIndexTracker; + } + + public NodeState getRoot() { + return root; + } + private static boolean isStatusChanged(NodeState before, NodeState after) { return !EqualsDiff.equals(before.getChildNode(STATUS_NODE), after.getChildNode(STATUS_NODE)); } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (revision 1851902) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (working copy) @@ -219,14 +219,17 @@ try{ if (node != null){ IndexDefinition defn = node.getDefinition(); - return Collections.singletonList(planBuilder(filter) - .setEstimatedEntryCount(defn.getFulltextEntryCount(node.getIndexStatistics().numDocs())) - .setCostPerExecution(defn.getCostPerExecution()) - .setCostPerEntry(defn.getCostPerEntry()) - .setDeprecated(defn.isDeprecated()) - .setAttribute(ATTR_INDEX_PATH, indexPath) - .setDeprecated(defn.isDeprecated()) - .build()); + LuceneIndexStatistics stats = node.getIndexStatistics(); + if (stats != null) { + return Collections.singletonList(planBuilder(filter) + .setEstimatedEntryCount(defn.getFulltextEntryCount(stats.numDocs())) + .setCostPerExecution(defn.getCostPerExecution()) + .setCostPerEntry(defn.getCostPerEntry()) + .setDeprecated(defn.isDeprecated()) + .setAttribute(ATTR_INDEX_PATH, indexPath) + .setDeprecated(defn.isDeprecated()) + .build()); + } } //No index node then no plan possible return Collections.emptyList(); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java (revision 1851902) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java (working copy) @@ -37,6 +37,7 @@ IndexSearcher getSearcher(); + @Nullable LuceneIndexStatistics getIndexStatistics(); List getPrimaryReaders(); Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (revision 1851902) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (working copy) @@ -29,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; @@ -46,11 +47,13 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.PerfLogger; import org.apache.jackrabbit.oak.plugins.index.lucene.util.fv.SimSearchUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; import org.apache.jackrabbit.oak.plugins.index.search.FieldNames; import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition; import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader; import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.spi.FulltextQueryTermsProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper; @@ -120,6 +123,8 @@ import org.apache.lucene.search.postingshighlight.PostingsHighlighter; import org.apache.lucene.search.spell.SuggestWord; import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.Version; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -186,6 +191,9 @@ */ public class LucenePropertyIndex extends FulltextIndex { + + private static boolean NON_LAZY = Boolean.getBoolean("oak.lucene.nonLazyIndex"); + private static double MIN_COST = 2.1; private static final Logger LOG = LoggerFactory @@ -689,7 +697,10 @@ @Override protected LuceneIndexNode acquireIndexNode(String indexPath) { - return tracker.acquireIndexNode(indexPath); + if (NON_LAZY) { + return tracker.acquireIndexNode(indexPath); + } + return new LazyLuceneIndexNode(tracker, indexPath); } @Override @@ -1552,4 +1563,128 @@ return null; } } + + /** + * A index node implementation that acquires the underlying index only if + * actually needed. This is to avoid downloading the index for the planning + * phase, if there is no chance that the index is actually used. + */ + static class LazyLuceneIndexNode implements LuceneIndexNode { + private AtomicBoolean released = new AtomicBoolean(); + private IndexTracker tracker; + private String indexPath; + private volatile LuceneIndexNode indexNode; + + LazyLuceneIndexNode(IndexTracker tracker, String indexPath) { + this.tracker = tracker; + this.indexPath = indexPath; + } + + @Override + public void release() { + if (released.getAndSet(true)) { + // already released + return; + } + if (indexNode != null) { + indexNode.release(); + } + // to ensure it is not used after releasing + indexNode = null; + tracker = null; + indexPath = null; + } + + private void checkNotReleased() { + if (released.get()) { + throw new IllegalStateException("Already released"); + } + } + + @Override + public LuceneIndexDefinition getDefinition() { + checkNotReleased(); + return tracker.getIndexDefinition(indexPath); + } + + private LuceneIndexNode getIndexNode() { + LuceneIndexNode n = findIndexNode(); + if (n == null) { + String message = "No index node, corrupt index? " + indexPath; + LOG.warn(message); + throw new IllegalStateException(message); + } + return n; + } + + @Nullable + private LuceneIndexNode findIndexNode() { + checkNotReleased(); + LuceneIndexNode n = indexNode; + // double checked locking implemented in the correct way for Java 5 + // and newer (actually I don't think this is ever called + // concurrently right now, but better be save) + if (n == null) { + synchronized (this) { + n = indexNode; + if (n == null) { + n = indexNode = tracker.acquireIndexNode(indexPath); + } + } + } + return n; + } + + @Override + public int getIndexNodeId() { + return getIndexNode().getIndexNodeId(); + } + + @Nullable + @Override + public LuceneIndexStatistics getIndexStatistics() { + LuceneIndexNode n = findIndexNode(); + if (n == null) { + return null; + } + return n.getIndexStatistics(); + } + + @Override + public IndexSearcher getSearcher() { + return getIndexNode().getSearcher(); + } + + @Override + public List getPrimaryReaders() { + return getIndexNode().getPrimaryReaders(); + } + + @Override + public @Nullable Directory getSuggestDirectory() { + return getIndexNode().getSuggestDirectory(); + } + + @Override + public List getNRTReaders() { + return getIndexNode().getNRTReaders(); + } + + @Override + public @Nullable AnalyzingInfixSuggester getLookup() { + return getIndexNode().getLookup(); + } + + @Override + public @Nullable LuceneIndexWriter getLocalWriter() throws IOException { + return getIndexNode().getLocalWriter(); + } + + @Override + public void refreshReadersOnWriteIfRequired() { + getIndexNode().refreshReadersOnWriteIfRequired(); + } + + } + } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java (revision 1851902) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java (working copy) @@ -197,7 +197,7 @@ } else if (pos < position) { LOG.warn("Seeking back on streaming index file {}. Current position {}, requested position {}. " + "Please make sure that CopyOnRead and prefetch of index files are enabled.", - getName(), position(), pos); + dirDetails + "/" + getName(), position(), pos); // seeking back on input stream. Close current one IOUtils.closeQuietly(blobInputStream); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (revision 1851902) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (working copy) @@ -24,10 +24,6 @@ import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; import static java.util.Arrays.asList; import static javax.jcr.PropertyType.TYPENAME_STRING; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertFalse; -import static junit.framework.Assert.assertTrue; -import static junit.framework.Assert.fail; import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM; import static org.apache.jackrabbit.JcrConstants.NT_BASE; @@ -54,6 +50,10 @@ import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PERSISTENCE_NAME; import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PERSISTENCE_PATH; import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.File; import java.io.IOException; Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (revision 1851902) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (working copy) @@ -1233,7 +1233,6 @@ assertQuery("select [jcr:path] from [nt:base] where propa is not null", asList("/test/a", "/test/b", "/test/c")); } - @Test public void rangeQueriesWithDate() throws Exception { Tree idx = createIndex("test1", of("propa", "propb")); @@ -2491,13 +2490,13 @@ prop1.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true); root.commit(); - //force CoR - executeQuery("SELECT * FROM [mix:title]", SQL2); + // force CoR + executeQuery("select * from [mix:title] where [jcr:title] = 'x'", SQL2); assertNotNull(corDir); String localPathBeforeReindex = corDir; - //CoW with re-indexing + // CoW with re-indexing idx.setProperty("reindex", true); root.commit(); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java (revision 1851902) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java (working copy) @@ -264,6 +264,7 @@ addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); createPath("/a").setProperty("foo", "bar"); root.commit(); @@ -277,8 +278,8 @@ assertQuery("select * from [nt:base] where [foo] = 'bar'", asList("/a", "/b")); - //Do multiple runs which lead to path being returned from both property and lucene - //index. But the actual result should only contain unique paths + // Do multiple runs which lead to path being returned from both property and lucene + // index. But the actual result should only contain unique paths runAsyncIndex(); runAsyncIndex(); @@ -326,6 +327,7 @@ addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); assertThat(explain("select * from [nt:base] where [jcr:content/foo] = 'bar'"), containsString("sync:(foo[jcr:content/foo] bar)")); @@ -333,7 +335,6 @@ containsString("sync:(foo bar)")); } - @Test public void relativePropertyTransform() throws Exception{ defnb.async("async", "nrt"); @@ -341,6 +342,7 @@ addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); createPath("/a/jcr:content").setProperty("foo", "bar"); createPath("/b").setProperty("foo", "bar"); @@ -360,6 +362,7 @@ indexPath = "/content/oak:index/fooIndex"; addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); createPath("/a").setProperty("foo", "bar"); createPath("/content/a").setProperty("foo", "bar"); @@ -427,7 +430,7 @@ @Test public void nodeTypeIndexing() throws Exception{ - registerTestNodTypes(); + registerTestNodeTypes(); defnb.async("async", "nrt"); defnb.nodeTypeIndex(); @@ -435,6 +438,7 @@ addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); createPath("/a", "oak:TestSuperType"); createPath("/b", "oak:TestTypeB"); @@ -448,7 +452,7 @@ @Test public void nodeType_mixins() throws Exception{ - registerTestNodTypes(); + registerTestNodeTypes(); defnb.async("async", "nrt"); defnb.nodeTypeIndex(); @@ -456,6 +460,7 @@ addIndex(indexPath, defnb); root.commit(); + runAsyncIndex(); createPath("/a", "oak:Unstructured", singletonList("oak:TestMixA")); createPath("/b", "oak:TestTypeB"); @@ -466,10 +471,10 @@ assertQuery("select * from [oak:TestMixA]", asList("/a", "/b")); } - private void registerTestNodTypes() throws IOException, CommitFailedException { + private void registerTestNodeTypes() throws IOException, CommitFailedException { optionalEditorProvider.delegate = new TypeEditorProvider(); NodeTypeRegistry.register(root, IOUtils.toInputStream(testNodeTypes, "utf-8"), "test nodeType"); - //Flush the changes to nodetypes + // Flush the changes to nodetypes root.commit(); } Index: oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java =================================================================== --- oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java (revision 1851902) +++ oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java (working copy) @@ -19,6 +19,8 @@ package org.apache.jackrabbit.oak.plugins.index.search; +import org.jetbrains.annotations.Nullable; + /** * Represents an instance of an index. * @@ -34,6 +36,7 @@ int getIndexNodeId(); + @Nullable IndexStatistics getIndexStatistics(); } Index: oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java =================================================================== --- oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java (revision 1851902) +++ oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java (working copy) @@ -105,6 +105,11 @@ } public IndexPlan getPlan() { + if (definition == null) { + log.debug("Index {} not loaded", indexPath); + return null; + } + IndexPlan.Builder builder = getPlanBuilder(); if (definition.isTestMode()){ @@ -281,6 +286,9 @@ costPerEntryFactor += sortOrder.size(); IndexPlan.Builder plan = defaultPlan(); + if (plan == null) { + return null; + } if (!sortOrder.isEmpty()) { plan.setSortOrder(sortOrder); } @@ -413,9 +421,16 @@ result.disableUniquePaths(); } - //If native function can be handled by this index then ensure - // that lowest cost if returned - return canHandleNativeFunction ? defaultPlan().setEstimatedEntryCount(1) : null; + if (!canHandleNativeFunction) { + return null; + } + IndexPlan.Builder b = defaultPlan(); + if (b == null) { + return null; + } + // If native function can be handled by this index, then ensure + // that lowest cost is returned + return b.setEstimatedEntryCount(1); } /* @@ -713,7 +728,16 @@ return rule.indexesAllNodesOfMatchingType() && !rule.isBasedOnNtBase(); } + @Nullable private IndexPlan.Builder defaultPlan() { + // With OAK-7947 lucene indexes return a non-null index node to delay reading index files + // While IndexNode could have a status check method but for now we are using this work-around + // to check null on {@code getIndexStatistics()} as proxy indicator + // (this could be avoided by returning lazy statistics) + if (indexNode.getIndexStatistics() == null) { + return null; + } + return new IndexPlan.Builder() .setCostPerExecution(definition.getCostPerExecution()) .setCostPerEntry(definition.getCostPerEntry()) @@ -730,11 +754,13 @@ } private long estimatedEntryCount() { - int numOfDocs = getNumDocs(); if (useActualEntryCount) { - return definition.isEntryCountDefined() ? definition.getEntryCount() : numOfDocs; + if (definition.isEntryCountDefined()) { + return definition.getEntryCount(); + } + return getNumDocs(); } else { - return estimatedEntryCount_Compat(numOfDocs); + return estimatedEntryCount_Compat(getNumDocs()); } } @@ -756,11 +782,20 @@ } private int getNumDocs() { - return indexNode.getIndexStatistics().numDocs(); + IndexStatistics indexStatistics = indexNode.getIndexStatistics(); + if (indexStatistics == null) { + log.warn("Statistics not available - possibly index is corrupt? Returning high doc count"); + return Integer.MAX_VALUE; + } + return indexStatistics.numDocs(); } private int getMaxPossibleNumDocs(Map propDefns, Filter filter) { IndexStatistics indexStatistics = indexNode.getIndexStatistics(); + if (indexStatistics == null) { + log.warn("Statistics not available - possibly index is corrupt? Returning high doc count"); + return Integer.MAX_VALUE; + } int minNumDocs = indexStatistics.numDocs(); for (Map.Entry propDef : propDefns.entrySet()) { String key = propDef.getKey();