Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java (revision 1635075) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndex.java (working copy) @@ -16,20 +16,11 @@ */ package org.apache.jackrabbit.oak.plugins.index.aggregate; - import java.util.Collections; -import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.NoSuchElementException; -import java.util.Set; import java.util.concurrent.atomic.AtomicReference; -import com.google.common.base.Function; -import com.google.common.base.Predicates; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; -import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.query.fulltext.FullTextAnd; import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression; import org.apache.jackrabbit.oak.query.fulltext.FullTextOr; @@ -38,11 +29,14 @@ import org.apache.jackrabbit.oak.query.index.FilterImpl; import org.apache.jackrabbit.oak.spi.query.Cursor; import org.apache.jackrabbit.oak.spi.query.Cursors; -import org.apache.jackrabbit.oak.spi.query.Cursors.AbstractCursor; import org.apache.jackrabbit.oak.spi.query.Filter; -import org.apache.jackrabbit.oak.spi.query.IndexRow; import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.google.common.base.Function; +import com.google.common.collect.Lists; + import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvanceFulltextQueryIndex; /** @@ -51,6 +45,8 @@ */ public class AggregateIndex implements AdvanceFulltextQueryIndex { + private static final Logger LOG = LoggerFactory.getLogger(AggregateIndex.class); + private final AdvanceFulltextQueryIndex baseIndex; public AggregateIndex(AdvanceFulltextQueryIndex baseIndex) { @@ -72,34 +68,47 @@ if (baseIndex == null) { return Collections.emptyList(); } - return baseIndex.getPlans(filter, sortOrder, rootState); +if (baseIndex.getClass().toString().indexOf("LucenePropertyIndex") >= 0) { +; + return Collections.emptyList(); +} + + FullTextExpression e = filter.getFullTextConstraint(); + if (getNodeAggregator() == null || e == null) { + // no aggregation: path-though + return baseIndex.getPlans(filter, sortOrder, rootState); + } + if (!hasCompositeExpression(e)) { + // path-though, but without node type restriction + return baseIndex.getPlans(newAggregationFilter(filter, null), sortOrder, rootState); + } + AggregateIndexPlan plan = new AggregateIndexPlan(filter); + collectCombinedPlan(e, filter, sortOrder, rootState, plan, ""); + if (plan.containsPathWithoutPlan()) { + // this is not expected (a full-text index that + // can't deal with a full-text restriction) + LOG.warn("Full-text index without plan: " + e); + return Collections.emptyList(); + } + return Collections.singletonList((IndexPlan) plan); } @Override public Cursor query(IndexPlan plan, NodeState rootState) { - - if (baseIndex.getNodeAggregator() == null) { + if (getNodeAggregator() == null) { + // path-though return baseIndex.query(plan, rootState); + } else if (!(plan instanceof AggregateIndexPlan)) { + // remove the node type restriction + plan.setFilter(newAggregationFilter(plan.getFilter(), null)); + return newAggregationCursor(plan, rootState); } - return newCursor(plan, baseIndex, rootState); + Filter filter = plan.getFilter(); + AggregateIndexPlan aggPlan = (AggregateIndexPlan) plan; + FullTextExpression constraint = filter.getFullTextConstraint(); + return flatten(constraint, aggPlan, filter, rootState, ""); } - private static Cursor newCursor(IndexPlan plan, AdvanceFulltextQueryIndex index, - NodeState state) { - Filter f = plan.getFilter(); - FullTextExpression e = f.getFullTextConstraint(); - if (hasCompositeExpression(e)) { - Cursor c = flatten(e, plan, index, state); - if (c != null) { - return c; - } - } - - IndexPlan newPlan = newPlanWithAggregationFilter(plan, null); - return new AggregationCursor(index.query(newPlan, - state), index.getNodeAggregator(), state); - } - private static boolean hasCompositeExpression(FullTextExpression ft) { if (ft == null) { return false; @@ -150,30 +159,34 @@ return hasNegative.get(); } - private static Cursor flatten(FullTextExpression constraint, - final IndexPlan plan, final AdvanceFulltextQueryIndex index, - final NodeState state) { + private Cursor flatten(FullTextExpression constraint, + final AggregateIndexPlan plan, + final Filter filter, final NodeState state, + final String path) { if (constraint == null) { return null; } - final Filter filter = plan.getFilter(); final AtomicReference result = new AtomicReference(); constraint.accept(new FullTextVisitor() { @Override public boolean visit(FullTextTerm term) { - result.set(filterToCursor(newPlanWithAggregationFilter(plan, term), - index, state)); + IndexPlan p = plan.getPlan(path); + result.set(newAggregationCursor(p, state)); return true; } @Override public boolean visit(FullTextAnd and) { Iterator iterator = and.list.iterator(); - Cursor c = flatten(iterator.next(), plan, index, state); + int index = 0; + Cursor c = flatten(iterator.next(), plan, filter, state, + path + " and(" + index + ")"); while (iterator.hasNext()) { + index++; FullTextExpression input = iterator.next(); - Cursor newC = flatten(input, plan, index, state); + Cursor newC = flatten(input, plan, filter, state, + path + " and(" + index + ")"); c = Cursors.newIntersectionCursor(c, newC, filter.getQueryEngineSettings()); } @@ -183,11 +196,13 @@ @Override public boolean visit(FullTextOr or) { + final int[] index = new int[1]; List cursors = Lists.transform(or.list, new Function() { @Override public Cursor apply(FullTextExpression input) { - return flatten(input, plan, index, state); + return flatten(input, plan, filter, state, + path + " or(" + index[0]++ + ")"); } }); result.set(Cursors.newConcatCursor(cursors, @@ -198,22 +213,63 @@ return result.get(); } - private static Cursor filterToCursor(IndexPlan plan, AdvanceFulltextQueryIndex index, - NodeState state) { - return new AggregationCursor(index.query(plan, state), - index.getNodeAggregator(), state); + private void collectCombinedPlan(FullTextExpression constraint, + final Filter filter, + final List sortOrder, + final NodeState state, + final AggregateIndexPlan target, + final String path) { + + constraint.accept(new FullTextVisitor() { + + @Override + public boolean visit(FullTextTerm term) { + List list = baseIndex.getPlans( + newAggregationFilter(filter, term), sortOrder, state); + target.setPlan(path, list); + return true; + } + + @Override + public boolean visit(FullTextAnd and) { + int index = 0; + for (FullTextExpression input : and.list) { + collectCombinedPlan(input, filter, sortOrder, state, target, + path + " and(" + index + ")"); + index++; + } + return true; + } + + @Override + public boolean visit(FullTextOr or) { + int index = 0; + for (FullTextExpression input : or.list) { + collectCombinedPlan(input, filter, sortOrder, state, target, + path + " or(" + index + ")"); + index++; + } + return true; + } + }); } - private static IndexPlan newPlanWithAggregationFilter(IndexPlan plan, FullTextExpression exp) { - FilterImpl f = new FilterImpl(plan.getFilter()); + private Cursor newAggregationCursor(IndexPlan plan, NodeState rootState) { + Cursor c = baseIndex.query(plan, rootState); + // we possibly get results from a child, + // so we need to wrap the cursor to do aggregation + return new AggregationCursor(c, + getNodeAggregator(), rootState); + } + + private static Filter newAggregationFilter(Filter filter, FullTextExpression exp) { + FilterImpl f = new FilterImpl(filter); // disables node type checks for now f.setMatchesAllTypes(true); if (exp != null) { f.setFullTextConstraint(exp); } - IndexPlan copy = plan.copy(); - copy.setFilter(f); - return copy; + return f; } @Override @@ -226,7 +282,23 @@ if (baseIndex == null) { return "aggregate no-index"; } - return "aggregate " + baseIndex.getPlanDescription(plan, root); + StringBuilder buff = new StringBuilder("aggregate "); + if (getNodeAggregator() == null) { + // path-though + buff.append(baseIndex.getPlanDescription(plan, root)); + return buff.toString(); + } else if (!(plan instanceof AggregateIndexPlan)) { + buff.append(baseIndex.getPlanDescription(plan, root)); + return buff.toString(); + } + AggregateIndexPlan aggPlan = (AggregateIndexPlan) plan; + for (IndexPlan p : aggPlan.getPlans()) { + if (p != null) { + buff.append(baseIndex.getPlanDescription(p, root)); + buff.append(" "); + } + } + return buff.toString(); } @Override @@ -242,101 +314,4 @@ return baseIndex.getNodeAggregator(); } - /** - * An aggregation aware cursor. - */ - private static class AggregationCursor extends AbstractCursor { - - private final Cursor cursor; - private final NodeAggregator aggregator; - private final NodeState rootState; - - private boolean init; - private boolean closed; - - /** - * the current row - */ - private IndexRow currentRow; - - /** - * the path of the current item of the cursor - */ - private String currentPath; - - /** - * all of the item's known aggregates - */ - private Iterator aggregates; - - /** - * should enforce uniqueness of the aggregated paths - */ - private Set seenPaths = new HashSet(); - - public AggregationCursor(Cursor cursor, NodeAggregator aggregator, - NodeState rootState) { - this.cursor = cursor; - this.aggregator = aggregator; - this.rootState = rootState; - } - - @Override - public boolean hasNext() { - if (!closed && !init) { - fetchNext(); - init = true; - } - return !closed; - } - - private void fetchNext() { - if (aggregates != null && aggregates.hasNext()) { - currentPath = aggregates.next(); - init = true; - return; - } - aggregates = null; - if (cursor.hasNext()) { - currentRow = cursor.next(); - String path = currentRow.getPath(); - aggregates = Iterators.filter(Iterators.concat( - Iterators.singletonIterator(path), - aggregator.getParents(rootState, path)), Predicates - .not(Predicates.in(seenPaths))); - fetchNext(); - return; - } - closed = true; - } - - @Override - public IndexRow next() { - if (!hasNext()) { - throw new NoSuchElementException(); - } - seenPaths.add(currentPath); - init = false; - if (currentRow.getPath().equals(currentPath)) { - return currentRow; - } - // create a new overlayed index row, - // where the path is different but all other - // properties are kept - return new IndexRow() { - - @Override - public String getPath() { - return currentPath; - } - - @Override - public PropertyValue getValue(String columnName) { - return currentRow.getValue(columnName); - } - - }; - } - } - } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexPlan.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexPlan.java (revision 0) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexPlan.java (working copy) @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.aggregate; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; + +import javax.annotation.CheckForNull; + +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +/** + * An index plan for multiple query indexes. + */ +public class AggregateIndexPlan implements IndexPlan { + + private Filter filter; + private boolean pathWithoutPlan; + + /** + * The map of terms to plans. + */ + private final HashMap basePlans = new HashMap(); + + AggregateIndexPlan(Filter filter) { + this.filter = filter; + } + + void setPlan(String path, List plans) { + if (plans.size() == 0) { + // no index + basePlans.put(path, null); + pathWithoutPlan = true; + } else { + // we always pick the first plan + basePlans.put(path, plans.get(0)); + } + } + + boolean containsPathWithoutPlan() { + return pathWithoutPlan; + } + + IndexPlan getPlan(String path) { + return basePlans.get(path); + } + + Collection getPlans() { + return basePlans.values(); + } + + @Override + public double getCostPerExecution() { + double cost = 0; + for (IndexPlan p : basePlans.values()) { + if (p != null) { + cost += p.getCostPerExecution(); + } + } + return cost; + } + + @Override + public double getCostPerEntry() { + // calculate the weigted average + double costPerEntry = 0; + long totalEntries = getEstimatedEntryCount(); + if (totalEntries == 0) { + return 0; + } + for (IndexPlan p : basePlans.values()) { + if (p != null) { + costPerEntry += p.getCostPerEntry() * p.getEstimatedEntryCount() / totalEntries; + } + } + return costPerEntry; + } + + @Override + public long getEstimatedEntryCount() { + long totalEntries = 0; + for (IndexPlan p : basePlans.values()) { + if (p != null) { + totalEntries += p.getEstimatedEntryCount(); + } + } + return totalEntries; + } + + @Override + public Filter getFilter() { + return filter; + } + + @Override + public void setFilter(Filter filter) { + this.filter = filter; + } + + /** + * Whether any base plan is delayed. + * + * @return true if yes + */ + @Override + public boolean isDelayed() { + for (IndexPlan p : basePlans.values()) { + if (p != null && p.isDelayed()) { + return true; + } + } + return false; + } + + /** + * Whether any base plan is a full text index. + * + * @return true if yes + */ + @Override + public boolean isFulltextIndex() { + for (IndexPlan p : basePlans.values()) { + if (p != null && p.isFulltextIndex()) { + return true; + } + } + return false; + } + + /** + * Whether all base plan include node data. + * + * @return true if yes + */ + @Override + public boolean includesNodeData() { + for (IndexPlan p : basePlans.values()) { + if (p != null && !p.includesNodeData()) { + return false; + } + } + return true; + } + + /** + * An aggregated query can not sort, as it gets results from a number of + * indexes. + * + * @return null + */ + @Override + public List getSortOrder() { + return null; + } + + // the following methods probably shouldn't be in the IndexPlan interface + // as they are only used locally (in the ordered index, or in the lucene index) + + @Override + @CheckForNull + public PropertyRestriction getPropertyRestriction() { + return null; + } + + @Override + public IndexPlan copy() { + return null; + } + + @Override + public NodeState getDefinition() { + return null; + } + + @Override + public String getPathPrefix() { + return null; + } + + @Override + @CheckForNull + public Object getAttribute(String name) { + return null; + } + +} Index: oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregationCursor.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregationCursor.java (revision 0) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregationCursor.java (working copy) @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.aggregate; + +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.query.IndexRow; +import org.apache.jackrabbit.oak.spi.query.Cursors.AbstractCursor; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import com.google.common.base.Predicates; +import com.google.common.collect.Iterators; + +/** + * An aggregation aware cursor. + */ +class AggregationCursor extends AbstractCursor { + + private final Cursor cursor; + private final NodeAggregator aggregator; + private final NodeState rootState; + + private boolean init; + private boolean closed; + + /** + * the current row + */ + private IndexRow currentRow; + + /** + * the path of the current item of the cursor + */ + private String currentPath; + + /** + * all of the item's known aggregates + */ + private Iterator aggregates; + + /** + * should enforce uniqueness of the aggregated paths + */ + private Set seenPaths = new HashSet(); + + public AggregationCursor(Cursor cursor, NodeAggregator aggregator, + NodeState rootState) { + this.cursor = cursor; + this.aggregator = aggregator; + this.rootState = rootState; + } + + @Override + public boolean hasNext() { + if (!closed && !init) { + fetchNext(); + init = true; + } + return !closed; + } + + private void fetchNext() { + if (aggregates != null && aggregates.hasNext()) { + currentPath = aggregates.next(); + init = true; + return; + } + aggregates = null; + if (cursor.hasNext()) { + currentRow = cursor.next(); + String path = currentRow.getPath(); + aggregates = Iterators.filter(Iterators.concat( + Iterators.singletonIterator(path), + aggregator.getParents(rootState, path)), Predicates + .not(Predicates.in(seenPaths))); + fetchNext(); + return; + } + closed = true; + } + + @Override + public IndexRow next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + seenPaths.add(currentPath); + init = false; + if (currentRow.getPath().equals(currentPath)) { + return currentRow; + } + // create a new overlayed index row, + // where the path is different but all other + // properties are kept + return new IndexRow() { + + @Override + public String getPath() { + return currentPath; + } + + @Override + public PropertyValue getValue(String columnName) { + return currentRow.getValue(columnName); + } + + }; + } +} \ No newline at end of file Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java (revision 1635075) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java (working copy) @@ -22,6 +22,7 @@ import static org.apache.jackrabbit.JcrConstants.NT_FILE; import static org.apache.jackrabbit.JcrConstants.NT_FOLDER; import static org.apache.jackrabbit.JcrConstants.JCR_DATA; +import static org.apache.jackrabbit.JcrConstants.NT_UNSTRUCTURED; import java.util.ArrayList; import java.util.Calendar; @@ -35,7 +36,10 @@ import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator; +import static org.apache.jackrabbit.oak.api.Type.NAME; +import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.plugins.memory.BinaryPropertyState.binaryProperty; +import static org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.NT_OAK_UNSTRUCTURED; import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent; import org.apache.jackrabbit.oak.query.AbstractQueryTest; @@ -400,4 +404,25 @@ "xpath", ImmutableList.of("/myFolder", "/myFolder/myFile", "/myFolder/myFile/jcr:content")); } + @Test + public void oak2226() throws Exception { + final String statement = "/jcr:root/content//element(*, nt:unstructured)[" + + "(jcr:contains(., 'mountain')) " + + "and " + + " (jcr:contains(jcr:content/metadata/@format, 'image'))" + + "]"; + Tree content = root.getTree("/").addChild("content"); + Tree expected = content.addChild("expected"); + expected.setProperty(JCR_PRIMARYTYPE, NT_UNSTRUCTURED, NAME); + expected.setProperty("title", "Lorem mountain ipsum", STRING); + Tree node = expected.addChild("jcr:content"); + node.setProperty(JCR_PRIMARYTYPE, NT_OAK_UNSTRUCTURED, NAME); + node = node.addChild("metadata"); + node.setProperty(JCR_PRIMARYTYPE, NT_OAK_UNSTRUCTURED, NAME); + node.setProperty("title", "Lorem mountain ipsum", STRING); + node.setProperty("format", "image/jpeg", STRING); + root.commit(); + + assertQuery(statement, "xpath", ImmutableList.of(expected.getPath())); + } }