### Eclipse Workspace Patch 1.0 #P oak-core Index: src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (working copy) @@ -486,6 +486,18 @@ } Iterator it = FilterIterators.newCombinedFilter(rowIt, distinct, limit, offset, orderBy, settings); + if (orderBy != null) { + // this will force the rows to be read, so that the size is known + it.hasNext(); + // we need the size, and there is no other way to get it right now + // but we also have to take limit and offset into account + long read = rowIt.getReadCount(); + // we will ignore whatever is behind 'limit+offset' + read = Math.min(saturatedAdd(limit, offset), read); + // and we will skip 'offset' entries + read = Math.max(0, read - offset); + size = read; + } if (measure) { // return the measuring iterator delegating the readCounts to the rowIterator it = new MeasuringIterator(this, it) { @@ -1091,6 +1103,10 @@ @Override public long getSize(SizePrecision precision, long max) { // Note: DISTINCT is ignored + if (size != -1) { + // "order by" was used, so we know the size + return size; + } return Math.min(limit, source.getSize(precision, max)); } #P oak-jcr Index: src/main/java/org/apache/jackrabbit/oak/jcr/package-info.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/package-info.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/package-info.java (working copy) @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -@Version("1.0") +@Version("1.1") @Export(optional = "provide:=true") package org.apache.jackrabbit.oak.jcr; Index: src/main/java/org/apache/jackrabbit/oak/jcr/query/QueryResultImpl.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/query/QueryResultImpl.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/query/QueryResultImpl.java (working copy) @@ -35,6 +35,7 @@ import org.apache.jackrabbit.oak.api.Result; import org.apache.jackrabbit.oak.api.ResultRow; import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.jcr.query.PrefetchIterator.PrefetchOptions; import org.apache.jackrabbit.oak.jcr.session.NodeImpl; import org.apache.jackrabbit.oak.jcr.session.SessionContext; import org.apache.jackrabbit.oak.jcr.delegate.NodeDelegate; @@ -50,24 +51,10 @@ static final Logger LOG = LoggerFactory.getLogger(QueryResultImpl.class); - /** - * The minimum number of rows / nodes to pre-fetch. - */ - private static final int PREFETCH_MIN = 20; - - /** - * The maximum number of rows / nodes to pre-fetch. - */ - private static final int PREFETCH_MAX = 100; - - /** - * The maximum number of milliseconds to prefetch rows / nodes. - */ - private static final int PREFETCH_TIMEOUT = 100; - + protected final SessionContext sessionContext; + final Result result; - private final SessionContext sessionContext; private final SessionDelegate sessionDelegate; public QueryResultImpl(SessionContext sessionContext, Result result) { @@ -136,8 +123,11 @@ }; final PrefetchIterator prefIt = new PrefetchIterator( sessionDelegate.sync(rowIterator), - PREFETCH_MIN, PREFETCH_TIMEOUT, PREFETCH_MAX, - result.getSize(), result); + new PrefetchOptions() { { + size = result.getSize(); + fastSize = sessionContext.getFastQueryResultSize(); + fastSizeCallback = result; + } }); return new RowIteratorAdapter(prefIt) { @Override public long getSize() { @@ -167,7 +157,8 @@ throw new RepositoryException("Query does not contain a selector: " + Arrays.toString(columnSelectorNames)); } - Iterator> nodeIterator = new Iterator>() { + Iterator> nodeIterator = + new Iterator>() { private final Iterator it = result.getRows().iterator(); private NodeImpl current; @@ -214,10 +205,14 @@ } }; - final PrefetchIterator> prefIt = new PrefetchIterator>( - sessionDelegate.sync(nodeIterator), - PREFETCH_MIN, PREFETCH_TIMEOUT, PREFETCH_MAX, - result.getSize(), result); + final PrefetchIterator> prefIt = + new PrefetchIterator>( + sessionDelegate.sync(nodeIterator), + new PrefetchOptions() { { + size = result.getSize(); + fastSize = sessionContext.getFastQueryResultSize(); + fastSizeCallback = result; + } }); return new NodeIteratorAdapter(prefIt) { @Override public long getSize() { Index: src/main/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIterator.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIterator.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIterator.java (working copy) @@ -24,6 +24,8 @@ import org.apache.jackrabbit.oak.api.Result; import org.apache.jackrabbit.oak.api.Result.SizePrecision; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * An iterator that pre-fetches a number of items in order to calculate the size @@ -37,12 +39,12 @@ */ public class PrefetchIterator implements Iterator { - private final boolean fastSize = Boolean.getBoolean("oak.fastQuerySize"); - - private final Result result; + static final Logger LOG = LoggerFactory.getLogger(PrefetchIterator.class); private final Iterator it; private final long minPrefetch, timeout, maxPrefetch; + private final boolean fastSize; + private final Result fastSizeCallback; private boolean prefetchDone; private Iterator prefetchIterator; private long size, position; @@ -51,23 +53,19 @@ * Create a new iterator. * * @param it the base iterator - * @param min the minimum number of items to pre-fetch - * @param timeout the maximum time to pre-fetch in milliseconds - * @param max the maximum number of items to pre-fetch - * @param size the size (prefetching is only required if -1) + * @param options the prefetch options to use * @param result (optional) the result to get the size from */ - PrefetchIterator(Iterator it, long min, long timeout, long max, - long size, Result result) { + PrefetchIterator(Iterator it, PrefetchOptions options) { this.it = it; - this.minPrefetch = min; - if (fastSize) { - timeout = 0; - } - this.timeout = timeout; - this.maxPrefetch = max; - this.size = size; - this.result = result; + this.minPrefetch = options.min; + this.maxPrefetch = options.max; + this.timeout = options.fastSize ? 0 : options.timeout; + this.fastSize = options.fastSize; + this.size = options.size; + this.fastSizeCallback = options.fastSizeCallback; +; + LOG.info("prefetch It " + minPrefetch + " max:" + maxPrefetch + " fast:" + fastSizeCallback + " size:" + size ); } @Override @@ -145,11 +143,52 @@ position -= list.size(); } if (size == -1 && fastSize) { - if (result != null) { - size = result.getSize(SizePrecision.EXACT, Long.MAX_VALUE); + if (fastSizeCallback != null) { + size = fastSizeCallback.getSize(SizePrecision.EXACT, Long.MAX_VALUE); } } return size; } + + /** + * The options to use for prefetching. + */ + public static class PrefetchOptions { + + // uses the "simple" named-parameter pattern + // see also http://stackoverflow.com/questions/1988016/named-parameter-idiom-in-java + + /** + * The minimum number of rows / nodes to pre-fetch. + */ + long min = 20; + + /** + * The maximum number of rows / nodes to pre-fetch. + */ + long max = 100; + + /** + * The maximum number of milliseconds to prefetch rows / nodes + * (ignored if fastSize is set). + */ + long timeout = 100; + + /** + * The size if known, or -1 if not (prefetching is only required if -1). + */ + long size; + + /** + * Whether or not the expected size should be read from the result. + */ + boolean fastSize; + + /** + * The result (optional) to get the size from, in case the fast size options is set. + */ + Result fastSizeCallback; + + } } Index: src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java (working copy) @@ -73,11 +73,6 @@ public class RepositoryImpl implements JackrabbitRepository { /** - * logger instance - */ - private static final Logger log = LoggerFactory.getLogger(RepositoryImpl.class); - - /** * Name of the session attribute value determining the session refresh * interval in seconds. * @@ -92,15 +87,21 @@ */ public static final String RELAXED_LOCKING = "oak.relaxed-locking"; + /** + * logger instance + */ + private static final Logger log = LoggerFactory.getLogger(RepositoryImpl.class); + + protected final Whiteboard whiteboard; private final GenericDescriptors descriptors; private final ContentRepository contentRepository; - protected final Whiteboard whiteboard; private final SecurityProvider securityProvider; private final int observationQueueLength; private final CommitRateLimiter commitRateLimiter; private final Clock clock; private final DelegatingGCMonitor gcMonitor = new DelegatingGCMonitor(); private final Registration gcMonitorRegistration; + private final boolean fastQueryResultSize; /** * {@link ThreadLocal} counter that keeps track of the save operations @@ -124,7 +125,8 @@ @Nonnull Whiteboard whiteboard, @Nonnull SecurityProvider securityProvider, int observationQueueLength, - CommitRateLimiter commitRateLimiter) { + CommitRateLimiter commitRateLimiter, + boolean fastQueryResultSize) { this.contentRepository = checkNotNull(contentRepository); this.whiteboard = checkNotNull(whiteboard); this.securityProvider = checkNotNull(securityProvider); @@ -134,6 +136,7 @@ this.statisticManager = new StatisticManager(whiteboard, scheduledExecutor); this.clock = new Clock.Fast(scheduledExecutor); this.gcMonitorRegistration = whiteboard.register(GCMonitor.class, gcMonitor, emptyMap()); + this.fastQueryResultSize = fastQueryResultSize; } //---------------------------------------------------------< Repository >--- @@ -322,7 +325,7 @@ Map attributes, SessionDelegate delegate, int observationQueueLength, CommitRateLimiter commitRateLimiter) { return new SessionContext(this, statisticManager, securityProvider, whiteboard, attributes, - delegate, observationQueueLength, commitRateLimiter); + delegate, observationQueueLength, commitRateLimiter, fastQueryResultSize); } /** Index: src/main/java/org/apache/jackrabbit/oak/jcr/osgi/RepositoryManager.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/osgi/RepositoryManager.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/osgi/RepositoryManager.java (working copy) @@ -78,6 +78,8 @@ private int observationQueueLength; private CommitRateLimiter commitRateLimiter; + + private boolean fastQueryResultSize; @Reference private SecurityProvider securityProvider; @@ -166,7 +168,7 @@ return bundleContext.registerService( Repository.class.getName(), new OsgiRepository(oak.createContentRepository(), whiteboard, securityProvider, - observationQueueLength, commitRateLimiter), + observationQueueLength, commitRateLimiter, fastQueryResultSize), new Properties()); } } Index: src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/Jcr.java (working copy) @@ -67,7 +67,8 @@ private SecurityProvider securityProvider; private int observationQueueLength = DEFAULT_OBSERVATION_QUEUE_LENGTH; - private CommitRateLimiter commitRateLimiter = null; + private CommitRateLimiter commitRateLimiter; + private boolean fastQueryResultSize; public Jcr(Oak oak) { this.oak = oak; @@ -198,13 +199,20 @@ return this; } + @Nonnull + public Jcr withFastQueryResultSize(boolean fastQueryResultSize) { + this.fastQueryResultSize = fastQueryResultSize; + return this; + } + public Repository createRepository() { return new RepositoryImpl( oak.createContentRepository(), oak.getWhiteboard(), securityProvider, observationQueueLength, - commitRateLimiter); + commitRateLimiter, + fastQueryResultSize); } } Index: src/main/java/org/apache/jackrabbit/oak/jcr/osgi/OsgiRepository.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/osgi/OsgiRepository.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/osgi/OsgiRepository.java (working copy) @@ -38,8 +38,10 @@ Whiteboard whiteboard, SecurityProvider securityProvider, int observationQueueLength, - CommitRateLimiter commitRateLimiter) { - super(repository, whiteboard, securityProvider, observationQueueLength, commitRateLimiter); + CommitRateLimiter commitRateLimiter, + boolean fastQueryResultSize) { + super(repository, whiteboard, securityProvider, observationQueueLength, + commitRateLimiter, fastQueryResultSize); } @Override Index: src/test/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIteratorTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIteratorTest.java (revision 1694443) +++ src/test/java/org/apache/jackrabbit/oak/jcr/query/PrefetchIteratorTest.java (working copy) @@ -24,19 +24,26 @@ import java.util.Iterator; import java.util.NoSuchElementException; +import org.apache.jackrabbit.oak.jcr.query.PrefetchIterator.PrefetchOptions; import org.junit.Test; /** * Test the PrefetchIterator class. */ public class PrefetchIteratorTest { - + @Test public void testKnownSize() { Iterable s; PrefetchIterator it; s = seq(0, 100); - it = new PrefetchIterator(s.iterator(), 5, 0, 10, 200, null); + it = new PrefetchIterator(s.iterator(), + new PrefetchOptions() { { + min = 5; + timeout = 0; + max = 10; + size = 200; + } }); // reports the 'wrong' value as it was set manually assertEquals(200, it.size()); } @@ -47,14 +54,26 @@ PrefetchIterator it; // long delay (10 ms per row) - long timeout = 10; + final long testTimeout = 10; s = seq(0, 100, 10); - it = new PrefetchIterator(s.iterator(), 5, timeout, 1000, -1, null); + it = new PrefetchIterator(s.iterator(), + new PrefetchOptions() { { + min = 5; + timeout = testTimeout; + max = 10; + size = -1; + } }); assertEquals(-1, it.size()); // no delay s = seq(0, 100); - it = new PrefetchIterator(s.iterator(), 5, timeout, 1000, -1, null); + it = new PrefetchIterator(s.iterator(), + new PrefetchOptions() { { + min = 5; + timeout = testTimeout; + max = 1000; + size = -1; + } }); assertEquals(100, it.size()); } @@ -65,10 +84,16 @@ for (int size : seq(0, 100)) { for (int readBefore : seq(0, 30)) { // every 3th time, use a timeout - long timeout = size % 3 == 0 ? 100 : 0; + final long testTimeout = size % 3 == 0 ? 100 : 0; Iterable s = seq(0, size); PrefetchIterator it = - new PrefetchIterator(s.iterator(), 20, timeout, 30, -1, null); + new PrefetchIterator(s.iterator(), + new PrefetchOptions() { { + min = 20; + timeout = testTimeout; + max = 30; + size = -1; + } }); for (int x : seq(0, readBefore)) { boolean hasNext = it.hasNext(); if (!hasNext) { @@ -80,7 +105,7 @@ assertEquals(m, x, it.next().intValue()); } String m = "s:" + size + " b:" + readBefore; - int max = timeout <= 0 ? 20 : 30; + int max = testTimeout <= 0 ? 20 : 30; if (size > max && readBefore <= size) { assertEquals(m, -1, it.size()); // calling it twice must not change the result Index: src/main/java/org/apache/jackrabbit/oak/jcr/session/SessionContext.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/jcr/session/SessionContext.java (revision 1694443) +++ src/main/java/org/apache/jackrabbit/oak/jcr/session/SessionContext.java (working copy) @@ -109,12 +109,15 @@ /** Paths of all session scoped locks held by this session. */ private final Set sessionScopedLocks = newHashSet(); + + private final boolean fastQueryResultSize; public SessionContext( @Nonnull Repository repository, @Nonnull StatisticManager statisticManager, @Nonnull SecurityProvider securityProvider, @Nonnull Whiteboard whiteboard, @Nonnull Map attributes, @Nonnull final SessionDelegate delegate, - int observationQueueLength, CommitRateLimiter commitRateLimiter) { + int observationQueueLength, CommitRateLimiter commitRateLimiter, + boolean fastQueryResultSize) { this.repository = checkNotNull(repository); this.statisticManager = statisticManager; this.securityProvider = checkNotNull(securityProvider); @@ -131,6 +134,7 @@ namespaces, delegate.getIdManager()); this.valueFactory = new ValueFactoryImpl( delegate.getRoot(), namePathMapper); + this.fastQueryResultSize = fastQueryResultSize; } public final Map getAttributes() { @@ -280,6 +284,13 @@ public Set getSessionScopedLocks() { return sessionScopedLocks; } + + public boolean getFastQueryResultSize() { + if (System.getProperty("oak.fastQuerySize") != null) { + return Boolean.getBoolean("oak.fastQuerySize"); + } + return fastQueryResultSize; + } //-----------------------------------------------------< NamePathMapper >--- #P oak-lucene Index: src/test/java/org/apache/jackrabbit/oak/jcr/query/ResultSizeTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/jcr/query/ResultSizeTest.java (revision 1694443) +++ src/test/java/org/apache/jackrabbit/oak/jcr/query/ResultSizeTest.java (working copy) @@ -86,7 +86,8 @@ StringBuilder buff; // fast (insecure) case - System.setProperty("oak.fastQuerySize", "true"); + // enabled by default now, in LuceneOakRepositoryStub + System.clearProperty("oak.fastQuerySize"); q = qm.createQuery(xpath, "xpath"); it = q.execute().getNodes(); result = it.getSize(); @@ -104,8 +105,10 @@ it = q.execute().getNodes(); assertEquals(90, it.getSize()); + // default (secure) case - System.clearProperty("oak.fastQuerySize"); + // manually disabled + System.setProperty("oak.fastQuerySize", "false"); q = qm.createQuery(xpath, "xpath"); it = q.execute().getNodes(); result = it.getSize(); @@ -117,7 +120,8 @@ } String regularResult = buff.toString(); assertEquals(regularResult, fastSizeResult); - + System.clearProperty("oak.fastQuerySize"); + } } \ No newline at end of file Index: src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (revision 1694443) +++ src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (working copy) @@ -58,6 +58,7 @@ new LuceneCompatModeInitializer("luceneGlobal", (Set) null)) .with((QueryIndexProvider)provider) .with((Observer) provider) + .withFastQueryResultSize(true) .with(new LuceneIndexEditorProvider()); }