From 1c54a751f28057bdeccbd450288814a17bc47e1f Mon Sep 17 00:00:00 2001 From: Vikas Saurabh Date: Mon, 18 Feb 2019 19:10:50 +0530 Subject: [PATCH] OAK-8046: Result items are not always correctly counted against the configured read limit if a query uses a lucene index --- .../oak/plugins/index/lucene/LuceneIndex.java | 20 ++++++++++++++++--- .../index/lucene/LucenePropertyIndex.java | 16 ++++++++++++--- .../index/search/spi/query/FulltextIndex.java | 11 +++++++++- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index b198cf4ba9..aa4dbc5dbe 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -299,7 +299,7 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { final boolean nonFullTextConstraints = parent.isEmpty(); final int parentDepth = getDepth(parent); QueryLimits settings = filter.getQueryLimits(); - Iterator itr = new AbstractIterator() { + LuceneResultRowIterator itr = new LuceneResultRowIterator() { private final Deque queue = Queues.newArrayDeque(); private final Set seenPaths = Sets.newHashSet(); private ScoreDoc lastDoc; @@ -316,6 +316,11 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { return endOfData(); } + @Override + public int rewoundCount() { + return reloadCount; + } + private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, String excerpt) throws IOException { IndexReader reader = searcher.getIndexReader(); PathStoredFieldVisitor visitor = new PathStoredFieldVisitor(); @@ -483,7 +488,7 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { throw new IllegalStateException("Too many version changes"); } lastDoc = null; - LOG.debug("Change in index version detected {} => {}. Query would be performed without " + + LOG.info("Change in index version detected {} => {}. Query would be performed without " + "offset; reload {}", currentVersion, lastSearchIndexerVersion, reloadCount); } this.lastSearchIndexerVersion = currentVersion; @@ -1146,12 +1151,13 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { private final SizeEstimator sizeEstimator; private long estimatedSize; - LucenePathCursor(final Iterator it, QueryLimits settings, SizeEstimator sizeEstimator, Filter filter) { + LucenePathCursor(final LuceneResultRowIterator it, QueryLimits settings, SizeEstimator sizeEstimator, Filter filter) { this.sizeEstimator = sizeEstimator; Iterator pathIterator = new Iterator() { private int readCount; + private int rewoundCount; @Override public boolean hasNext() { @@ -1160,6 +1166,10 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { @Override public String next() { + if (it.rewoundCount() > rewoundCount) { + readCount = 0; + rewoundCount = it.rewoundCount(); + } currentRow = it.next(); readCount++; if (readCount % TRAVERSING_WARNING == 0) { @@ -1230,4 +1240,8 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { return estimatedSize = sizeEstimator.getSize(); } } + + static abstract class LuceneResultRowIterator extends AbstractIterator { + abstract int rewoundCount(); + } } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index 36a9d2be47..e3692eca51 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -251,7 +251,7 @@ public class LucenePropertyIndex extends FulltextIndex { final Sort sort = getSort(plan); final PlanResult pr = getPlanResult(plan); QueryLimits settings = filter.getQueryLimits(); - Iterator itr = new AbstractIterator() { + LuceneResultRowIterator rItr = new LuceneResultRowIterator() { private final Deque queue = Queues.newArrayDeque(); private final Set seenPaths = Sets.newHashSet(); private ScoreDoc lastDoc; @@ -260,6 +260,7 @@ public class LucenePropertyIndex extends FulltextIndex { private IndexSearcher indexSearcher; private int indexNodeId = -1; private LuceneFacetProvider facetProvider = null; + private int rewoundCount = 0; @Override protected FulltextResultRow computeNext() { @@ -270,6 +271,11 @@ public class LucenePropertyIndex extends FulltextIndex { return endOfData(); } + @Override + public int rewoundCount() { + return rewoundCount; + } + private FulltextResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, Map excerpts, LuceneFacetProvider facetProvider, String explanation) throws IOException { @@ -535,7 +541,8 @@ public class LucenePropertyIndex extends FulltextIndex { if (indexNodeId != indexNode.getIndexNodeId()){ //if already initialized then log about change if (indexNodeId > 0){ - LOG.debug("Change in index version detected. Query would be performed without offset"); + LOG.info("Change in index version detected. Query would be performed without offset"); + rewoundCount++; } indexSearcher = indexNode.getSearcher(); @@ -550,13 +557,14 @@ public class LucenePropertyIndex extends FulltextIndex { indexSearcher = null; } }; + Iterator itr = rItr; SizeEstimator sizeEstimator = getSizeEstimator(plan); if (pr.hasPropertyIndexResult() || pr.evaluateSyncNodeTypeRestriction()) { itr = mergePropertyIndexResult(plan, rootState, itr); } - return new FulltextPathCursor(itr, plan, settings, sizeEstimator); + return new FulltextPathCursor(itr, rItr, plan, settings, sizeEstimator); } private static Query addDescendantClauseIfRequired(Query query, IndexPlan plan) { @@ -1687,4 +1695,6 @@ public class LucenePropertyIndex extends FulltextIndex { } + static abstract class LuceneResultRowIterator extends AbstractIterator implements IteratorRewoundStateProvider { + } } diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java index 6d15584a0d..097f59511f 100644 --- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java +++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java @@ -374,12 +374,13 @@ public abstract class FulltextIndex implements AdvancedQueryIndex, QueryIndex, N private long estimatedSize; private final int numberOfFacets; - public FulltextPathCursor(final Iterator it, final IndexPlan plan, QueryLimits settings, SizeEstimator sizeEstimator) { + public FulltextPathCursor(final Iterator it, final IteratorRewoundStateProvider rIt, final IndexPlan plan, QueryLimits settings, SizeEstimator sizeEstimator) { pathPrefix = plan.getPathPrefix(); this.sizeEstimator = sizeEstimator; Iterator pathIterator = new Iterator() { private int readCount; + private int rewoundCount; @Override public boolean hasNext() { @@ -388,6 +389,10 @@ public abstract class FulltextIndex implements AdvancedQueryIndex, QueryIndex, N @Override public String next() { + if (rIt.rewoundCount() > rewoundCount) { + readCount = 0; + rewoundCount = rIt.rewoundCount(); + } currentRow = it.next(); readCount++; if (readCount % TRAVERSING_WARNING == 0) { @@ -493,6 +498,10 @@ public abstract class FulltextIndex implements AdvancedQueryIndex, QueryIndex, N } } + public interface IteratorRewoundStateProvider { + int rewoundCount(); + } + /** * A query result facet, composed by its label and count. */ -- 2.19.1