From 1b0df881f48cfc84395cc76981c04cd8fdc47e45 Mon Sep 17 00:00:00 2001 From: Joel Richard Date: Fri, 14 Aug 2015 15:55:17 +0200 Subject: [PATCH] OAK-2679 - Reduce execution plan overhead * Add QueryIndex.getMinimalCost which allows to skip the remaining index cost comparison if it finds an index whose cost is <= the minimum cost of the next more expensive index * Implement getMinimalCost for all indexes * Do not read index definitions in PropertyIndex.getCost if the filter does not have property restrictions and selector restrictions * Stop looking for better index definition in PropertyIndex once it finds a definition with the absolute minimum cost * Cache for 30 seconds if there are no ordered property or Solr indexes --- .../plugins/index/aggregate/AggregateIndex.java | 5 +++ .../oak/plugins/index/aggregate/package-info.java | 2 +- .../oak/plugins/index/diffindex/UUIDDiffIndex.java | 5 +++ .../oak/plugins/index/nodetype/NodeTypeIndex.java | 6 ++++ .../index/nodetype/NodeTypeIndexLookup.java | 5 +++ .../index/property/OrderedPropertyIndex.java | 20 +++++++++-- .../index/property/OrderedPropertyIndexLookup.java | 24 ++++++++++--- .../property/OrderedPropertyIndexProvider.java | 41 ++++++++++++++++++++- .../oak/plugins/index/property/PropertyIndex.java | 42 +++++++++++++++++++--- .../index/property/PropertyIndexLookup.java | 2 +- .../plugins/index/property/PropertyIndexPlan.java | 6 +++- .../plugins/index/reference/ReferenceIndex.java | 9 ++++- .../org/apache/jackrabbit/oak/query/QueryImpl.java | 24 ++++++++++++- .../oak/query/index/TraversingIndex.java | 5 +++ .../jackrabbit/oak/spi/query/QueryIndex.java | 11 +++++- .../LowCostOrderedPropertyIndexProvider.java | 13 +++++-- .../index/property/OrderedIndexCostTest.java | 24 ++++++------- .../property/OrderedPropertyIndexQueryTest.java | 6 ++-- .../apache/jackrabbit/oak/jcr/query/QueryTest.java | 15 +++++++- .../oak/plugins/index/lucene/LuceneIndex.java | 5 +++ .../plugins/index/lucene/LucenePropertyIndex.java | 5 +++ .../index/lucene/LowCostLuceneIndexProvider.java | 8 +++-- .../plugins/index/solr/query/SolrQueryIndex.java | 9 ++++- .../index/solr/query/SolrQueryIndexProvider.java | 24 +++++++++++++ 24 files changed, 277 insertions(+), 39 deletions(-) diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java index df90ddf..6f19507 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java @@ -55,6 +55,11 @@ public class AggregateIndex implements AdvanceFulltextQueryIndex { } @Override + public double getMinimumCost() { + return baseIndex.getMinimumCost(); + } + + @Override public double getCost(Filter filter, NodeState rootState) { throw new UnsupportedOperationException("Not supported as implementing AdvancedQueryIndex"); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/package-info.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/package-info.java index 488b670..482a92a 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/package-info.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/package-info.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("1.1.0") +@Version("1.2.0") @Export(optional = "provide:=true") package org.apache.jackrabbit.oak.plugins.index.aggregate; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diffindex/UUIDDiffIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diffindex/UUIDDiffIndex.java index 4c534c3..21436b5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diffindex/UUIDDiffIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/diffindex/UUIDDiffIndex.java @@ -25,6 +25,11 @@ public class UUIDDiffIndex extends DiffIndex { } @Override + public double getMinimumCost() { + return 0; + } + + @Override public String getIndexName() { return "UUIDDiffIndex"; } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java index 47cc03f..5cea862 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndex.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.index.nodetype; import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexLookup; import org.apache.jackrabbit.oak.spi.query.Cursor; import org.apache.jackrabbit.oak.spi.query.Cursors; import org.apache.jackrabbit.oak.spi.query.Filter; @@ -36,6 +37,11 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; class NodeTypeIndex implements QueryIndex, JcrConstants { @Override + public double getMinimumCost() { + return NodeTypeIndexLookup.MINIMUM_COST; + } + + @Override public double getCost(Filter filter, NodeState root) { // TODO don't call getCost for such queries if (filter.getFullTextConstraint() != null) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndexLookup.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndexLookup.java index e4a61bb..0186ab3 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndexLookup.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/nodetype/NodeTypeIndexLookup.java @@ -31,6 +31,11 @@ import com.google.common.collect.Iterables; */ class NodeTypeIndexLookup implements JcrConstants { + /** + * Derived from {@link #getCost(Filter)} + */ + static final double MINIMUM_COST = 2 * PropertyIndexLookup.COST_OVERHEAD; + private final NodeState root; public NodeTypeIndexLookup(NodeState root) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndex.java index e21bdc6..dd8aca5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndex.java @@ -44,13 +44,29 @@ public class OrderedPropertyIndex implements QueryIndex, AdvancedQueryIndex { private static final Logger LOG = LoggerFactory.getLogger(OrderedPropertyIndex.class); + /** + * we're local. Low-cost + */ + private static final double COST_PER_EXECUTION = 1; + + private final OrderedPropertyIndexProvider indexProvider; + + public OrderedPropertyIndex(OrderedPropertyIndexProvider indexProvider) { + this.indexProvider = indexProvider; + } + + @Override + public double getMinimumCost() { + return COST_PER_EXECUTION; + } + @Override public String getIndexName() { return TYPE; } OrderedPropertyIndexLookup getLookup(NodeState root) { - return new OrderedPropertyIndexLookup(root); + return new OrderedPropertyIndexLookup(indexProvider, root); } /** @@ -68,7 +84,7 @@ public class OrderedPropertyIndex implements QueryIndex, AdvancedQueryIndex { */ static IndexPlan.Builder getIndexPlanBuilder(final Filter filter) { IndexPlan.Builder b = new IndexPlan.Builder(); - b.setCostPerExecution(1); // we're local. Low-cost + b.setCostPerExecution(COST_PER_EXECUTION); // we're local but slightly more expensive than a standard PropertyIndex b.setCostPerEntry(1.3); b.setFulltextIndex(false); // we're never full-text diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexLookup.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexLookup.java index 4d20241..5e9e6f5 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexLookup.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexLookup.java @@ -76,18 +76,21 @@ public class OrderedPropertyIndexLookup { */ private static final int MAX_COST = 100; + private final OrderedPropertyIndexProvider indexProvider; + private NodeState root; private String name; private OrderedPropertyIndexLookup parent; - public OrderedPropertyIndexLookup(NodeState root) { - this(root, "", null); + public OrderedPropertyIndexLookup(OrderedPropertyIndexProvider indexProvider, NodeState root) { + this(indexProvider, root, "", null); } - public OrderedPropertyIndexLookup(NodeState root, String name, + public OrderedPropertyIndexLookup(OrderedPropertyIndexProvider indexProvider, NodeState root, String name, OrderedPropertyIndexLookup parent) { + this.indexProvider = indexProvider; this.root = root; this.name = name; this.parent = parent; @@ -106,9 +109,14 @@ public class OrderedPropertyIndexLookup { */ @Nullable NodeState getIndexNode(NodeState node, String propertyName, Filter filter) { + if (parent == null && !indexProvider.mayHaveRootIndexes()) { + return null; + } + // keep a fallback to a matching index def that has *no* node type constraints // (initially, there is no fallback) NodeState fallback = null; + boolean hasRootIndexes = false; NodeState state = node.getChildNode(INDEX_DEFINITIONS_NAME); for (ChildNodeEntry entry : state.getChildNodeEntries()) { @@ -117,6 +125,8 @@ public class OrderedPropertyIndexLookup { if (type == null || type.isArray() || !getType().equals(type.getValue(Type.STRING))) { continue; } + hasRootIndexes = true; + if (contains(index.getNames(PROPERTY_NAMES), propertyName)) { NodeState indexContent = index.getChildNode(INDEX_CONTENT_NODE_NAME); if (!indexContent.exists()) { @@ -140,6 +150,10 @@ public class OrderedPropertyIndexLookup { } } } + + if (parent == null) { + indexProvider.indicateRootIndexes(hasRootIndexes); + } return fallback; } @@ -243,7 +257,7 @@ public class OrderedPropertyIndexLookup { for (String element : PathUtils.elements(path)) { if (lookup == null) { lookup = new OrderedPropertyIndexLookup( - root.getChildNode(element), element, this); + indexProvider, root.getChildNode(element), element, this); } else { remainder = PathUtils.concat(remainder, element); } @@ -308,7 +322,7 @@ public class OrderedPropertyIndexLookup { for (String element : PathUtils.elements(path)) { if (lookup == null) { lookup = new OrderedPropertyIndexLookup( - root.getChildNode(element), element, this); + indexProvider, root.getChildNode(element), element, this); } else { remainder = PathUtils.concat(remainder, element); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexProvider.java index ab6d137..cb993c8 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexProvider.java @@ -18,6 +18,8 @@ package org.apache.jackrabbit.oak.plugins.index.property; import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; @@ -33,9 +35,46 @@ import com.google.common.collect.ImmutableList; @Service(QueryIndexProvider.class) public class OrderedPropertyIndexProvider implements QueryIndexProvider { + private static final long DEFAULT_NO_INDEX_CACHE_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + + /** + * How often it should check for a new ordered property index. + */ + private static long noIndexCacheTimeout = DEFAULT_NO_INDEX_CACHE_TIMEOUT; + + /** + * The last time when it checked for the existence of an ordered property index AND could not find any. + */ + private volatile long lastNegativeIndexCheck = 0; + @Override @Nonnull public List getQueryIndexes(NodeState nodeState) { - return ImmutableList. of(new OrderedPropertyIndex()); + return ImmutableList. of(new OrderedPropertyIndex(this)); + } + + /** + * @return true if there may be any ordered indexes below the root path + */ + boolean mayHaveRootIndexes() { + return System.currentTimeMillis() - lastNegativeIndexCheck > noIndexCacheTimeout; + } + + /** + * Indicates whether or not there are ordered indexes below the root path + * + * @param hasRootIndexes + */ + void indicateRootIndexes(boolean hasRootIndexes) { + lastNegativeIndexCheck = hasRootIndexes ? 0 : System.currentTimeMillis(); + } + + public static void setCacheTimeoutForTesting(long timeout) { + noIndexCacheTimeout = timeout; } + + public static void resetCacheTimeoutForTesting() { + noIndexCacheTimeout = DEFAULT_NO_INDEX_CACHE_TIMEOUT; + } + } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java index 0e2b50b..893a955 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndex.java @@ -98,6 +98,11 @@ class PropertyIndex implements QueryIndex { private static final Logger LOG = LoggerFactory.getLogger(PropertyIndex.class); + /** + * Cached property index plan + */ + private PropertyIndexPlan plan; + static Set encode(PropertyValue value) { if (value == null) { return null; @@ -121,7 +126,22 @@ class PropertyIndex implements QueryIndex { return values; } - private static PropertyIndexPlan plan(NodeState root, Filter filter) { + private PropertyIndexPlan getPlan(NodeState root, Filter filter) { + // Reuse cached plan if the filter is the same (which should always be the case). The filter is compared as a + // string because it would not be possible to use its equals method since the preparing flag would be different + // and creating a separate isSimilar method is not worth the effort since it would not be used anymore once the + // PropertyIndex has been refactored to an AdvancedQueryIndex (which will make the plan cache obsolete). + PropertyIndexPlan plan = this.plan; + if (plan != null && plan.getFilter().toString().equals(filter.toString())) { + return plan; + } else { + plan = createPlan(root, filter); + this.plan = plan; + return plan; + } + } + + private static PropertyIndexPlan createPlan(NodeState root, Filter filter) { PropertyIndexPlan bestPlan = null; // TODO support indexes on a path @@ -138,6 +158,10 @@ class PropertyIndex implements QueryIndex { plan.getName(), plan.getCost()); if (bestPlan == null || plan.getCost() < bestPlan.getCost()) { bestPlan = plan; + // Stop comparing if the costs are the minimum + if (plan.getCost() == PropertyIndexPlan.COST_OVERHEAD) { + break; + } } } } @@ -148,6 +172,12 @@ class PropertyIndex implements QueryIndex { //--------------------------------------------------------< QueryIndex >-- + + @Override + public double getMinimumCost() { + return PropertyIndexPlan.COST_OVERHEAD; + } + @Override public String getIndexName() { return PROPERTY; @@ -163,8 +193,12 @@ class PropertyIndex implements QueryIndex { // not an appropriate index for native search return Double.POSITIVE_INFINITY; } + if (filter.getPropertyRestrictions().isEmpty() && filter.getSelector().getSelectorConstraints().isEmpty()) { + // not an appropriate index for no property restrictions & selector constraints + return Double.POSITIVE_INFINITY; + } - PropertyIndexPlan plan = plan(root, filter); + PropertyIndexPlan plan = getPlan(root, filter); if (plan != null) { return plan.getCost(); } else { @@ -174,7 +208,7 @@ class PropertyIndex implements QueryIndex { @Override public Cursor query(Filter filter, NodeState root) { - PropertyIndexPlan plan = plan(root, filter); + PropertyIndexPlan plan = getPlan(root, filter); checkState(plan != null, "Property index is used even when no index" + " is available for filter " + filter); @@ -183,7 +217,7 @@ class PropertyIndex implements QueryIndex { @Override public String getPlan(Filter filter, NodeState root) { - PropertyIndexPlan plan = plan(root, filter); + PropertyIndexPlan plan = getPlan(root, filter); if (plan != null) { return plan.toString(); } else { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexLookup.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexLookup.java index 684a597..ab77580 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexLookup.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexLookup.java @@ -63,7 +63,7 @@ public class PropertyIndexLookup { /** * The cost overhead to use the index in number of read operations. */ - private static final double COST_OVERHEAD = 2; + public static final double COST_OVERHEAD = 2; /** * The maximum cost when the index can be used. diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexPlan.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexPlan.java index 3efbb65..aebf952 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexPlan.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/PropertyIndexPlan.java @@ -57,7 +57,7 @@ public class PropertyIndexPlan { /** * The cost overhead to use the index in number of read operations. */ - private static final double COST_OVERHEAD = 2; + static final double COST_OVERHEAD = 2; /** * The maximum cost when the index can be used. @@ -255,6 +255,10 @@ public class PropertyIndexPlan { return cursor; } + Filter getFilter() { + return filter; + } + //------------------------------------------------------------< Object >-- @Override diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java index bb0a787..c9be0b9 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/reference/ReferenceIndex.java @@ -53,6 +53,13 @@ class ReferenceIndex implements QueryIndex { private static final ContentMirrorStoreStrategy STORE = new ContentMirrorStoreStrategy(); + private static final double COST = 1; + + @Override + public double getMinimumCost() { + return COST; + } + @Override public String getIndexName() { return NAME; @@ -72,7 +79,7 @@ class ReferenceIndex implements QueryIndex { for (PropertyRestriction pr : filter.getPropertyRestrictions()) { if (isEqualityRestrictionOnType(pr, REFERENCE) || isEqualityRestrictionOnType(pr, WEAKREFERENCE)) { - return 1; + return COST; } } // not an appropriate index diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java index 8d47b8a..69d4021 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java @@ -16,6 +16,7 @@ package org.apache.jackrabbit.oak.query; import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -28,6 +29,7 @@ import com.google.common.collect.AbstractIterator; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.google.common.collect.Ordering; import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; @@ -122,6 +124,13 @@ public class QueryImpl implements Query { private static final Logger LOG = LoggerFactory.getLogger(QueryImpl.class); + private static final Ordering MINIMAL_COST_ORDERING = new Ordering() { + @Override + public int compare(QueryIndex left, QueryIndex right) { + return Double.compare(left.getMinimumCost(), right.getMinimumCost()); + } + }; + SourceImpl source; final String statement; final HashMap bindVariableMap = new HashMap(); @@ -920,7 +929,16 @@ public class QueryImpl implements Query { double bestCost = Double.POSITIVE_INFINITY; IndexPlan bestPlan = null; - for (QueryIndex index : indexProvider.getQueryIndexes(rootState)) { + + // Sort the indexes according to their minimum cost to be able to skip the remaining indexes if the cost of the + // current index is below the minimum cost of the next index. + final List queryIndexes = MINIMAL_COST_ORDERING + .sortedCopy(indexProvider.getQueryIndexes(rootState)); + + for (int i = 0; i < queryIndexes.size(); i++) { + final QueryIndex index = queryIndexes.get(i); + final QueryIndex nextIndex = (i < queryIndexes.size()) ? queryIndexes.get(i) : null; + double cost; String indexName = index.getIndexName(); IndexPlan indexPlan = null; @@ -990,6 +1008,10 @@ public class QueryImpl implements Query { bestIndex = index; bestPlan = indexPlan; } + // Stop looking for a better index if the current best cost is lower than the next minimum cost + if (nextIndex != null && bestCost <= nextIndex.getMinimumCost()) { + break; + } } if (traversalEnabled) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/index/TraversingIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/index/TraversingIndex.java index 80705a1..e09ae96 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/index/TraversingIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/index/TraversingIndex.java @@ -34,6 +34,11 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; public class TraversingIndex implements QueryIndex { @Override + public double getMinimumCost() { + return 0; + } + + @Override public Cursor query(Filter filter, NodeState rootState) { return Cursors.newTraversingCursor(filter, rootState); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/QueryIndex.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/QueryIndex.java index 490c507..f1f85e8 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/QueryIndex.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/query/QueryIndex.java @@ -53,7 +53,16 @@ import static org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; * even thought it will be set in the filter. */ public interface QueryIndex { - + + /** + * Returns the minimum cost which {@link #getCost(Filter, NodeState)} would return in the best possible case. + *

+ * The implementation should return a static/cached value because it is called very often. + * + * @return the minimum cost for the index + */ + double getMinimumCost(); + /** * Estimate the worst-case cost to query with the given filter. The returned * cost is a value between 1 (very fast; lookup of a unique node) and the diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/LowCostOrderedPropertyIndexProvider.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/LowCostOrderedPropertyIndexProvider.java index 69b4c13..f4d3347 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/LowCostOrderedPropertyIndexProvider.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/LowCostOrderedPropertyIndexProvider.java @@ -30,13 +30,22 @@ import com.google.common.collect.ImmutableList; public class LowCostOrderedPropertyIndexProvider extends OrderedPropertyIndexProvider { @Override public List getQueryIndexes(NodeState nodeState) { - return ImmutableList. of(new LowCostOrderedPropertyIndex()); + return ImmutableList. of(new LowCostOrderedPropertyIndex(this)); } private static class LowCostOrderedPropertyIndex extends OrderedPropertyIndex { + public LowCostOrderedPropertyIndex(LowCostOrderedPropertyIndexProvider indexProvider) { + super(indexProvider); + } + @Override - public double getCost(Filter filter, NodeState root) { + public double getMinimumCost() { return 1e-3; } + + @Override + public double getCost(Filter filter, NodeState root) { + return getMinimumCost(); + } } } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedIndexCostTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedIndexCostTest.java index 37fa3c5..25540c7 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedIndexCostTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedIndexCostTest.java @@ -92,7 +92,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costFullTextConstraint() { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeState root = InitialContent.INITIAL_CONTENT; Filter filter = EasyMock.createNiceMock(Filter.class); FullTextExpression fte = EasyMock.createNiceMock(FullTextExpression.class); @@ -106,7 +106,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costContainsNativeConstraints(){ - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeState root = InitialContent.INITIAL_CONTENT; Filter filter = EasyMock.createNiceMock(Filter.class); EasyMock.expect(filter.containsNativeConstraint()).andReturn(true).anyTimes(); @@ -124,7 +124,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { */ @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costGreaterThanAscendingDirection() throws Exception { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineAscendingIndex(builder); NodeState root = builder.getNodeState(); @@ -148,7 +148,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { */ @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costGreaterThanEqualAscendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineAscendingIndex(builder); NodeState root = builder.getNodeState(); @@ -173,7 +173,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { */ @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costLessThanAscendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineAscendingIndex(builder); NodeState root = builder.getNodeState(); @@ -192,7 +192,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costLessThanEqualsAscendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineAscendingIndex(builder); NodeState root = builder.getNodeState(); @@ -212,7 +212,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costGreaterThanDescendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineDescendingIndex(builder); NodeState root = builder.getNodeState(); @@ -232,7 +232,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costGreaterEqualThanDescendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineDescendingIndex(builder); NodeState root = builder.getNodeState(); @@ -253,7 +253,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costLessThanDescendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineDescendingIndex(builder); NodeState root = builder.getNodeState(); @@ -273,7 +273,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costLessThanEqualDescendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineDescendingIndex(builder); NodeState root = builder.getNodeState(); @@ -294,7 +294,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costBetweenDescendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineDescendingIndex(builder); NodeState root = builder.getNodeState(); @@ -317,7 +317,7 @@ public class OrderedIndexCostTest extends BasicOrderedPropertyIndexQueryTest { @Test @Ignore("As of OAK-622 this should no longer be used. Removing later.") public void costBetweenAscendingDirection() throws IllegalArgumentException, RepositoryException { - OrderedPropertyIndex index = new OrderedPropertyIndex(); + OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); NodeBuilder builder = InitialContent.INITIAL_CONTENT.builder(); defineAscendingIndex(builder); NodeState root = builder.getNodeState(); diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexQueryTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexQueryTest.java index 96173ca..3f6e114 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexQueryTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/property/OrderedPropertyIndexQueryTest.java @@ -507,7 +507,7 @@ public class OrderedPropertyIndexQueryTest extends BasicOrderedPropertyIndexQuer NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - final OrderedPropertyIndex index = new OrderedPropertyIndex(); + final OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); final String nodeTypeName = JcrConstants.NT_BASE; Filter filter = createFilter(indexed, nodeTypeName); @@ -583,7 +583,7 @@ public class OrderedPropertyIndexQueryTest extends BasicOrderedPropertyIndexQuer NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - final OrderedPropertyIndex index = new OrderedPropertyIndex(); + final OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); final String nodeTypeName = JcrConstants.NT_BASE; Filter filter = createFilter(indexed, nodeTypeName); @@ -624,7 +624,7 @@ public class OrderedPropertyIndexQueryTest extends BasicOrderedPropertyIndexQuer NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); - final OrderedPropertyIndex index = new OrderedPropertyIndex(); + final OrderedPropertyIndex index = new OrderedPropertyIndex(new OrderedPropertyIndexProvider()); final String nodeTypeName = JcrConstants.NT_BASE; FilterImpl filter = createFilter(indexed, nodeTypeName); filter.restrictProperty("somethingNotIndexed", Operator.EQUAL, PropertyValues.newLong(1L)); diff --git a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryTest.java b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryTest.java index f263200..dd0ac4e 100644 --- a/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryTest.java +++ b/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/query/QueryTest.java @@ -56,6 +56,9 @@ import org.apache.jackrabbit.commons.JcrUtils; import org.apache.jackrabbit.commons.cnd.CndImporter; import org.apache.jackrabbit.oak.jcr.AbstractRepositoryTest; import org.apache.jackrabbit.oak.jcr.NodeStoreFixture; +import org.apache.jackrabbit.oak.plugins.index.property.OrderedPropertyIndexProvider; +import org.junit.After; +import org.junit.Before; import org.junit.Ignore; import org.junit.Test; @@ -67,7 +70,17 @@ public class QueryTest extends AbstractRepositoryTest { public QueryTest(NodeStoreFixture fixture) { super(fixture); } - + + @Before + public void disableCaching() { + OrderedPropertyIndexProvider.setCacheTimeoutForTesting(0); + } + + @After + public void enableCaching() { + OrderedPropertyIndexProvider.resetCacheTimeoutForTesting(); + } + @Test public void typeConversion() throws Exception { Session session = getAdminSession(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index fabec74..17ff4dc 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -177,6 +177,11 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { } @Override + public double getMinimumCost() { + return 4; // TODO discuss + } + + @Override public String getIndexName() { return "lucene"; } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index 7bb2759..2492d01 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -190,6 +190,11 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati } @Override + public double getMinimumCost() { + return 3; // TODO discuss + } + + @Override public String getIndexName() { return "lucene-property"; } diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java index 1a96e77..318381a 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java @@ -36,19 +36,23 @@ public class LowCostLuceneIndexProvider extends LuceneIndexProvider { } private static class LowCostLuceneIndex extends LuceneIndex { - public LowCostLuceneIndex(IndexTracker tracker, NodeAggregator aggregator) { super(tracker, aggregator); } @Override + public double getMinimumCost() { + return 1e-3; + } + + @Override public List getPlans(Filter filter, List sortOrder, NodeState rootState) { String indexPath = new LuceneIndexLookup(rootState).getOldFullTextIndexPath(filter, tracker); if (indexPath == null){ return Collections.emptyList(); } return Collections.singletonList(planBuilder(filter) - .setCostPerExecution(1e-3) + .setCostPerExecution(getMinimumCost()) .setAttribute(ATTR_INDEX_PATH, indexPath) .build()); } diff --git a/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java b/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java index ff6e1f7..83d995d 100644 --- a/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java +++ b/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java @@ -76,6 +76,8 @@ public class SolrQueryIndex implements FulltextQueryIndex, QueryIndex.AdvanceFul static final String NATIVE_LUCENE_QUERY = "native*lucene"; + private static double COST_FOR_SINGLE_RESTRICTION = 10; + private final Logger log = LoggerFactory.getLogger(SolrQueryIndex.class); private final String name; @@ -103,6 +105,11 @@ public class SolrQueryIndex implements FulltextQueryIndex, QueryIndex.AdvanceFul } @Override + public double getMinimumCost() { + return COST_FOR_SINGLE_RESTRICTION / 4; // divided by number of matches in getMatchingFilterRestrictions + } + + @Override public String getIndexName() { return name; } @@ -110,7 +117,7 @@ public class SolrQueryIndex implements FulltextQueryIndex, QueryIndex.AdvanceFul @Override public double getCost(Filter filter, NodeState root) { // cost is inverse proportional to the number of matching restrictions, infinite if no restriction matches - double cost = 10d / getMatchingFilterRestrictions(filter); + double cost = COST_FOR_SINGLE_RESTRICTION / getMatchingFilterRestrictions(filter); if (log.isDebugEnabled()) { log.debug("Solr: cost for {} is {}", name, cost); } diff --git a/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java b/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java index 8253dcb..a364dfc 100644 --- a/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java +++ b/oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndexProvider.java @@ -17,9 +17,11 @@ package org.apache.jackrabbit.oak.plugins.index.solr.query; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.WeakHashMap; +import java.util.concurrent.TimeUnit; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -49,6 +51,11 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPER */ public class SolrQueryIndexProvider implements QueryIndexProvider { + /** + * How often it should check for a new Solr index. + */ + private static final long NO_INDEX_CACHE_TIMEOUT = TimeUnit.SECONDS.toMillis(30); + private final Logger log = LoggerFactory.getLogger(getClass()); private final SolrServerProvider solrServerProvider; @@ -59,6 +66,11 @@ public class SolrQueryIndexProvider implements QueryIndexProvider { private final Map estimators = new WeakHashMap(); + /** + * The last time when it checked for the existence of a Solr index AND could not find any. + */ + private volatile long lastNegativeIndexCheck = 0; + public SolrQueryIndexProvider(@Nonnull SolrServerProvider solrServerProvider, @Nonnull OakSolrConfigurationProvider oakSolrConfigurationProvider, @Nullable NodeAggregator nodeAggregator) { this.oakSolrConfigurationProvider = oakSolrConfigurationProvider; @@ -73,7 +85,19 @@ public class SolrQueryIndexProvider implements QueryIndexProvider { @Nonnull @Override public List getQueryIndexes(NodeState nodeState) { + List queryIndexes; + // Return immediately if there are not any Solr indexes and the cache timeout has not been exceeded + long currentTime = System.currentTimeMillis(); + if (currentTime - lastNegativeIndexCheck > NO_INDEX_CACHE_TIMEOUT) { + queryIndexes = internalGetQueryIndexes(nodeState); + lastNegativeIndexCheck = queryIndexes.isEmpty() ? currentTime : 0; + } else { + queryIndexes = Collections.emptyList(); + } + return queryIndexes; + } + private List internalGetQueryIndexes(NodeState nodeState) { List tempIndexes = new ArrayList(); NodeState definitions = nodeState.getChildNode(INDEX_DEFINITIONS_NAME); for (ChildNodeEntry entry : definitions.getChildNodeEntries()) { -- 2.3.6