Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (date 1433167150000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (date 1433224669000) @@ -13,15 +13,9 @@ */ package org.apache.jackrabbit.oak.query; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Set; - +import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; @@ -82,6 +76,16 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + /** * Represents a parsed query. */ @@ -468,7 +472,7 @@ logDebug("query execute " + statement); logDebug("query plan " + getPlan()); } - RowIterator rowIt = new RowIterator(context.getBaseState()); + final RowIterator rowIt = new RowIterator(context.getBaseState()); Comparator orderBy; boolean sortUsingIndex = false; // TODO add issue about order by optimization for multiple selectors @@ -509,41 +513,34 @@ } else { orderBy = ResultRowImpl.getComparator(orderings); } - Iterator it = + Iterator it = FilterIterators.newCombinedFilter(rowIt, distinct, limit, offset, orderBy, settings); if (measure) { - // run the query - while (it.hasNext()) { - it.next(); + // return the measuring iterator delegating the readCounts to the rowIterator + it = new MeasuringIterator(this, it) { + @Override + protected void setColumns(ColumnImpl[] col) { + columns = col; - } + } - columns = new ColumnImpl[] { - new ColumnImpl("measure", "selector", "selector"), - new ColumnImpl("measure", "scanCount", "scanCount") - }; - ArrayList list = new ArrayList(); - ResultRowImpl r = new ResultRowImpl(this, - Tree.EMPTY_ARRAY, - new PropertyValue[] { - PropertyValues.newString("query"), - PropertyValues.newLong(rowIt.getReadCount()) - }, - null, null); - list.add(r); + + @Override + protected long getReadCount() { + return rowIt.getReadCount(); + } + + @Override + protected Map getSelectorScanCount() { + Map selectorReadCounts = Maps.newHashMap(); - for (SelectorImpl selector : selectors) { + for (SelectorImpl selector : selectors) { - r = new ResultRowImpl(this, - Tree.EMPTY_ARRAY, - new PropertyValue[] { - PropertyValues.newString(selector.getSelectorName()), - PropertyValues.newLong(selector.getScanCount()), - }, - null, null); - list.add(r); + selectorReadCounts.put(selector.getSelectorName(), selector.getScanCount()); - } + } - it = list.iterator(); + return selectorReadCounts; - } + } + }; + } return it; } - + @Override public String getPlan() { return source.getPlan(context.getBaseState()); @@ -599,7 +596,7 @@ } estimatedCost = result.prepare().getEstimatedCost(); source = result; - + } private static SourceImpl buildJoin(SourceImpl result, SourceImpl last, List conditions) { @@ -632,6 +629,98 @@ */ Filter createFilter(boolean preparing) { return source.createFilter(preparing); + } + + + /** + * Abstract decorating iterator for measure queries. The iterator delegates to the underlying actual + * query iterator to lazily execute and return counts. + */ + static abstract class MeasuringIterator extends AbstractIterator { + private Iterator delegate; + private Query query; + private List results; + private boolean init; + + MeasuringIterator(Query query, Iterator delegate) { + this.query = query; + this.delegate = delegate; + results = Lists.newArrayList(); + } + + @Override + protected ResultRowImpl computeNext() { + if (!init) { + getRows(); + } + + if (!results.isEmpty()) { + return results.remove(0); + } else { + return endOfData(); + } + } + + void getRows() { + // run the query + while (delegate.hasNext()) { + delegate.next(); + } + + ColumnImpl[] columns = new ColumnImpl[] { + new ColumnImpl("measure", "selector", "selector"), + new ColumnImpl("measure", "scanCount", "scanCount") + }; + setColumns(columns); + + ResultRowImpl r = new ResultRowImpl(query, + Tree.EMPTY_ARRAY, + new PropertyValue[] { + PropertyValues.newString("query"), + PropertyValues.newLong(getReadCount()) + }, + null, null); + results.add(r); + + Map selectorScanCount = getSelectorScanCount(); + for (String selector : selectorScanCount.keySet()) { + r = new ResultRowImpl(query, + Tree.EMPTY_ARRAY, + new PropertyValue[] { + PropertyValues.newString(selector), + PropertyValues.newLong(selectorScanCount.get(selector)), + }, + null, null); + results.add(r); + } + init = true; + } + + /** + * Set the measure specific columns in the query object + * @param columns the measure specific columns + */ + protected abstract void setColumns(ColumnImpl[] columns); + + /** + * Retrieve the selector scan count + * @return map of selector to scan count + */ + protected abstract Map getSelectorScanCount(); + + /** + * Retrieve the query read count + * @return count + */ + protected abstract long getReadCount(); + + /** + * Retrieves the actual query iterator + * @return the delegate + */ + protected Iterator getDelegate() { + return delegate; + } } /** Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java (date 1433167150000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/UnionQueryImpl.java (date 1433224669000) @@ -19,12 +19,16 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; +import com.google.common.collect.Maps; + import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.api.Result; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.query.ast.ColumnImpl; import org.apache.jackrabbit.oak.query.ast.OrderingImpl; +import org.apache.jackrabbit.oak.query.QueryImpl.MeasuringIterator; import org.apache.jackrabbit.oak.spi.query.PropertyValues; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -127,6 +131,9 @@ @Override public ColumnImpl[] getColumns() { + if (columns != null) { + return columns; + } return left.getColumns(); } @@ -239,14 +246,60 @@ LOG.debug("query union plan {}", getPlan()); } } - Iterator it = Iterators.concat(left.getRows(), right.getRows()); - if (measure) { - // both queries measure themselves - return it; - } boolean distinct = !unionAll; Comparator orderBy = ResultRowImpl.getComparator(orderings); - it = FilterIterators.newCombinedFilter(it, distinct, limit, offset, orderBy, settings); + + Iterator it; + final Iterator leftRows = left.getRows(); + final Iterator rightRows = right.getRows(); + Iterator leftIter = leftRows; + Iterator rightIter = rightRows; + + // if measure retrieve the backing delegate iterator instead + if (measure) { + leftIter = ((MeasuringIterator) leftRows).getDelegate(); + rightIter = ((MeasuringIterator) rightRows).getDelegate(); + } + + it = FilterIterators + .newCombinedFilter(Iterators.concat(leftIter, rightIter), distinct, limit, offset, orderBy, settings); + + if (measure) { + // return the measuring iterator for the union + it = new MeasuringIterator(this, it) { + MeasuringIterator left = (MeasuringIterator) leftRows; + MeasuringIterator right = (MeasuringIterator) rightRows; + + @Override + protected void setColumns(ColumnImpl[] cols) { + columns = cols; + left.setColumns(cols); + right.setColumns(cols); + } + + @Override + protected Map getSelectorScanCount() { + // Merge the 2 maps from the left and right queries to get the selector counts + Map leftSelectorScan = left.getSelectorScanCount(); + Map rightSelectorScan = right.getSelectorScanCount(); + Map unionScan = Maps.newHashMap(leftSelectorScan); + for (String key : rightSelectorScan.keySet()) { + if (unionScan.containsKey(key)) { + unionScan.put(key, rightSelectorScan.get(key) + unionScan.get(key)); + } else { + unionScan.put(key, rightSelectorScan.get(key)); + } + } + return unionScan; + } + + @Override + protected long getReadCount() { + return left.getReadCount() + right.getReadCount(); + } + }; + } + return it; } @@ -254,5 +307,4 @@ public void setInternal(boolean isInternal) { this.isInternal = isInternal; } - } Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java (date 1433167150000) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/xpath/Statement.java (date 1433224669000) @@ -28,8 +28,8 @@ */ public class Statement { - private boolean explain; - private boolean measure; + boolean explain; + boolean measure; /** * The selector to get the columns from (the selector used in the select @@ -52,9 +52,6 @@ public Statement optimize() { ignoreOrderByScoreDesc(); - if (explain || measure) { - return this; - } if (where == null) { return this; } @@ -80,6 +77,9 @@ } union.orderList = orderList; union.xpathQuery = xpathQuery; + union.measure = measure; + union.explain = explain; + return union; } @@ -249,6 +249,12 @@ @Override public String toString() { StringBuilder buff = new StringBuilder(); + // explain | measure ... + if (this.explain) { + buff.append("explain "); + } else if (measure) { + buff.append("measure "); + } buff.append(s1).append(" union ").append(s2); // order by ... if (orderList != null && !orderList.isEmpty()) { Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (date 1433167150000) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (date 1433224669000) @@ -1358,6 +1358,46 @@ assertQuery(queryString, "xpath", asList("/test/b")); } + @Test + public void unionSortResultCount() throws Exception { + // Index Definition + Tree idx = createIndex("test1", of("propa", "propb", "propc")); + idx.setProperty(createProperty(ORDERED_PROP_NAMES, of("propc"), STRINGS)); + useV2(idx); + + // create test data + Tree test = root.getTree("/").addChild("test"); + root.commit(); + + List nodes = Lists.newArrayList(); + Random r = new Random(); + int seed = -2; + for (int i = 0; i < 1000; i++) { + Tree a = test.addChild("a" + i); + a.setProperty("propa", "fooa"); + seed += 2; + int num = 0 + r.nextInt(100); + a.setProperty("propc", num); + nodes.add(num); + } + + seed = -1; + for (int i = 0; i < 1000; i++) { + Tree a = test.addChild("b" + i); + a.setProperty("propb", "foob"); + seed += 2; + int num = 100 + r.nextInt(100); + a.setProperty("propc", num); + nodes.add(num); + } + root.commit(); + + // scan count scans the whole result set + String query = + "measure /jcr:root//element(*, nt:base)[(@propa = 'fooa' or @propb = 'foob')] order by @propc"; + assertThat(measureWithLimit(query, XPATH, 100), containsString("scanCount: 200")); + } + private Tree createFileNode(Tree tree, String name, String content, String mimeType){ return createFileNode(tree, name, new ArrayBasedBlob(content.getBytes()), mimeType); }