null.
*/
public FilteredQuery (Query query, Filter filter) {
- if (query == null || filter == null)
- throw new IllegalArgumentException("Query and filter cannot be null.");
- this.query = query;
- this.filter = filter;
+ this(query, filter, null);
}
/**
- * Expert: decides if a filter should be executed as "random-access" or not.
- * random-access means the filter "filters" in a similar way as deleted docs are filtered
- * in lucene. This is faster when the filter accepts many documents.
- * However, when the filter is very sparse, it can be faster to execute the query+filter
- * as a conjunction in some cases.
- *
- * The default implementation returns true if the first document accepted by the
- * filter is < 100.
+ * Expert: Constructs a new query which applies a filter to the results of the original query.
+ * {@link Filter#getDocIdSet} will be called every time this query is used in a search.
+ * @param query Query to be filtered, cannot be null.
+ * @param filter Filter to apply to query results, cannot be null.
+ * @param strategy a filter strategy used to create a filtered scorer.
*
- * @lucene.internal
+ * @see FilterStrategy
*/
- protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
- return firstFilterDoc < 100;
+ public FilteredQuery (Query query, Filter filter, FilterStrategy strategy) {
+ if (query == null || filter == null)
+ throw new IllegalArgumentException("Query and filter cannot be null.");
+ this.strategy = strategy == null ? RANDOM_ACCESS_FILTER_STRATEGY : strategy;
+ this.query = query;
+ this.filter = filter;
}
-
+
/**
* Returns a Weight that applies the filter to the enclosed query's Weight.
* This is accomplished by overriding the Scorer returned by the Weight.
@@ -121,7 +120,7 @@ public class FilteredQuery extends Query {
// return a filtering scorer
@Override
- public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, final Bits acceptDocs) throws IOException {
assert filter != null;
final DocIdSet filterDocIdSet = filter.getDocIdSet(context, acceptDocs);
@@ -129,109 +128,184 @@ public class FilteredQuery extends Query {
// this means the filter does not accept any documents.
return null;
}
+ return strategy.filteredScorer(context, scoreDocsInOrder, topScorer, weight, filterDocIdSet);
- final DocIdSetIterator filterIter = filterDocIdSet.iterator();
- if (filterIter == null) {
- // this means the filter does not accept any documents.
- return null;
- }
+ }
+ };
+ }
+
+ /**
+ * A Scorer that uses a "leap-frog" approach (also called "zig-zag join"). The scorer and the filter
+ * take turns trying to advance to each other's next matching document, often
+ * jumping past the target document. When both land on the same document, it's
+ * collected.
+ */
+ private static final class LeapFrogScorer extends Scorer {
+ private final Scorer scorer;
+ private final DocIdSetIterator filterIter;
+ private final int firstFilteredDoc;
+ private int scorerDoc = -1;
+ private int filterDoc;
- final int firstFilterDoc = filterIter.nextDoc();
- if (firstFilterDoc == DocIdSetIterator.NO_MORE_DOCS) {
- return null;
+ protected LeapFrogScorer(Weight weight, int firstFilteredDoc, DocIdSetIterator filterIter, Scorer other) {
+ super(weight);
+ this.filterDoc = this.firstFilteredDoc = firstFilteredDoc;
+ this.scorer = other;
+ this.filterIter = filterIter;
+ }
+
+ // optimization: we are topScorer and collect directly using short-circuited algo
+ @Override
+ public void score(Collector collector) throws IOException {
+ int filterDoc = firstFilteredDoc;
+ int scorerDoc = scorer.advance(filterDoc);
+ // the normalization trick already applies the boost of this query,
+ // so we can use the wrapped scorer directly:
+ collector.setScorer(scorer);
+ for (;;) {
+ if (scorerDoc == filterDoc) {
+ // Check if scorer has exhausted, only before collecting.
+ if (scorerDoc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
+ collector.collect(scorerDoc);
+ filterDoc = filterIter.nextDoc();
+ scorerDoc = scorer.advance(filterDoc);
+ } else if (scorerDoc > filterDoc) {
+ filterDoc = filterIter.advance(scorerDoc);
+ } else {
+ scorerDoc = scorer.advance(filterDoc);
}
-
- final Bits filterAcceptDocs = filterDocIdSet.bits();
- final boolean useRandomAccess = (filterAcceptDocs != null && FilteredQuery.this.useRandomAccess(filterAcceptDocs, firstFilterDoc));
-
- if (useRandomAccess) {
- // if we are using random access, we return the inner scorer, just with other acceptDocs
- return weight.scorer(context, scoreDocsInOrder, topScorer, filterAcceptDocs);
+ }
+ }
+
+ private int advanceToNextCommonDoc() throws IOException {
+ for (;;) {
+ if (scorerDoc < filterDoc) {
+ scorerDoc = scorer.advance(filterDoc);
+ } else if (scorerDoc == filterDoc) {
+ return scorerDoc;
} else {
- assert firstFilterDoc > -1;
- // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
- // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
- final Scorer scorer = weight.scorer(context, true, false, null);
- return (scorer == null) ? null : new Scorer(this) {
- private int scorerDoc = -1, filterDoc = firstFilterDoc;
-
- // optimization: we are topScorer and collect directly using short-circuited algo
- @Override
- public void score(Collector collector) throws IOException {
- int filterDoc = firstFilterDoc;
- int scorerDoc = scorer.advance(filterDoc);
- // the normalization trick already applies the boost of this query,
- // so we can use the wrapped scorer directly:
- collector.setScorer(scorer);
- for (;;) {
- if (scorerDoc == filterDoc) {
- // Check if scorer has exhausted, only before collecting.
- if (scorerDoc == DocIdSetIterator.NO_MORE_DOCS) {
- break;
- }
- collector.collect(scorerDoc);
- filterDoc = filterIter.nextDoc();
- scorerDoc = scorer.advance(filterDoc);
- } else if (scorerDoc > filterDoc) {
- filterDoc = filterIter.advance(scorerDoc);
- } else {
- scorerDoc = scorer.advance(filterDoc);
- }
- }
- }
-
- private int advanceToNextCommonDoc() throws IOException {
- for (;;) {
- if (scorerDoc < filterDoc) {
- scorerDoc = scorer.advance(filterDoc);
- } else if (scorerDoc == filterDoc) {
- return scorerDoc;
- } else {
- filterDoc = filterIter.advance(scorerDoc);
- }
- }
- }
+ filterDoc = filterIter.advance(scorerDoc);
+ }
+ }
+ }
- @Override
- public int nextDoc() throws IOException {
- // don't go to next doc on first call
- // (because filterIter is already on first doc):
- if (scorerDoc != -1) {
- filterDoc = filterIter.nextDoc();
- }
- return advanceToNextCommonDoc();
- }
-
- @Override
- public int advance(int target) throws IOException {
- if (target > filterDoc) {
- filterDoc = filterIter.advance(target);
- }
- return advanceToNextCommonDoc();
- }
+ @Override
+ public int nextDoc() throws IOException {
+ // don't go to next doc on first call
+ // (because filterIter is already on first doc):
+ if (scorerDoc != -1) {
+ filterDoc = filterIter.nextDoc();
+ }
+ return advanceToNextCommonDoc();
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > filterDoc) {
+ filterDoc = filterIter.advance(target);
+ }
+ return advanceToNextCommonDoc();
+ }
- @Override
- public int docID() {
- return scorerDoc;
- }
-
- @Override
- public float score() throws IOException {
- return scorer.score();
- }
-
- @Override
- public float freq() throws IOException { return scorer.freq(); }
-
- @Override
- public Collectiontrue. Otherwise this strategy falls back to a "zig-zag join" (
+ * {@link FilteredQuery#LEAP_FROG_FILTER_STRATEGY}) strategy.
+ *
+ * + * Note: this strategy is the default strategy in {@link FilteredQuery} + *
+ */ + public static final FilterStrategy RANDOM_ACCESS_FILTER_STRATEGY = new RandomAccessFilterStrategy(); + + /** + * A filter strategy that uses a "leap-frog" approach (also called "zig-zag join"). + * The scorer and the filter + * take turns trying to advance to each other's next matching document, often + * jumping past the target document. When both land on the same document, it's + * collected. + */ + public static final FilterStrategy LEAP_FROG_FILTER_STRATEGY = new LeapFrogFilterStragey(); + + /** + * A filter strategy that advances the {@link Scorer} first and consults the + * {@link DocIdSet} for each matched document. + *+ * Use this strategy if the filter computation is more expensive than document + * scoring or if the filter has a linear running time to compute the next + * matching doc like exact geo distances. + *
+ */ + public static final FilterStrategy DOC_FIRST_FILTER_STRATEGY = new DocFirstFilterStrategy(); + + /** Abstract class that defines how the filter ({@link DocIdSet}) applied during document collection. */ + public static abstract class FilterStrategy { + + /** + * Returns a filtered {@link Scorer} based on this strategy. + * + * @param context + * the {@link AtomicReaderContext} for which to return the {@link Scorer}. + * @param scoreDocsInOrder + * specifies whether in-order scoring of documents is required. Note + * that if set to false (i.e., out-of-order scoring is required), + * this method can return whatever scoring mode it supports, as every + * in-order scorer is also an out-of-order one. However, an + * out-of-order scorer may not support {@link Scorer#nextDoc()} + * and/or {@link Scorer#advance(int)}, therefore it is recommended to + * request an in-order scorer if use of these methods is required. + * @param topScorer + * if true, {@link Scorer#score(Collector)} will be called; if false, + * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will + * be called. + * @param weight the {@link FilteredQuery} {@link Weight} to create the filtered scorer. + * @param docIdSet the filter {@link DocIdSet} to apply + * @return a filtered scorer + * + * @throws IOException if an {@link IOException} occurs + */ + public abstract Scorer filteredScorer(AtomicReaderContext context, + boolean scoreDocsInOrder, boolean topScorer, Weight weight, + DocIdSet docIdSet) throws IOException; + } + + /** + * A {@link FilterStrategy} that conditionally uses a random access filter if + * the given {@link DocIdSet} supports random access (returns a non-null value + * from {@link DocIdSet#bits()}) and + * {@link RandomAccessFilterStrategy#useRandomAccess(Bits, int)} returns + *true. Otherwise this strategy falls back to a "zig-zag join" (
+ * {@link FilteredQuery#LEAP_FROG_FILTER_STRATEGY}) strategy .
+ */
+ public static class RandomAccessFilterStrategy extends FilterStrategy {
+
+ @Override
+ public Scorer filteredScorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Weight weight, DocIdSet docIdSet) throws IOException {
+ final DocIdSetIterator filterIter = docIdSet.iterator();
+ if (filterIter == null) {
+ // this means the filter does not accept any documents.
+ return null;
+ }
+
+ final int firstFilterDoc = filterIter.nextDoc();
+ if (firstFilterDoc == DocIdSetIterator.NO_MORE_DOCS) {
+ return null;
+ }
+
+ final Bits filterAcceptDocs = docIdSet.bits();
+ // force if RA is requested
+ final boolean useRandomAccess = (filterAcceptDocs != null && (useRandomAccess(filterAcceptDocs, firstFilterDoc)));
+ if (useRandomAccess) {
+ // if we are using random access, we return the inner scorer, just with other acceptDocs
+ return weight.scorer(context, scoreDocsInOrder, topScorer, filterAcceptDocs);
+ } else {
+ assert firstFilterDoc > -1;
+ // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
+ // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
+ final Scorer scorer = weight.scorer(context, true, false, null);
+ return (scorer == null) ? null : new LeapFrogScorer(weight, firstFilterDoc, filterIter, scorer);
+ }
+ }
+
+ /**
+ * Expert: decides if a filter should be executed as "random-access" or not.
+ * random-access means the filter "filters" in a similar way as deleted docs are filtered
+ * in lucene. This is faster when the filter accepts many documents.
+ * However, when the filter is very sparse, it can be faster to execute the query+filter
+ * as a conjunction in some cases.
+ *
+ * The default implementation returns true if the first document accepted by the
+ * filter is < 100.
+ *
+ * @lucene.internal
+ */
+ protected boolean useRandomAccess(Bits bits, int firstFilterDoc) {
+ return firstFilterDoc < 100;
+ }
+ }
+
+ private static final class LeapFrogFilterStragey extends FilterStrategy {
+
+ @Override
+ public Scorer filteredScorer(AtomicReaderContext context,
+ boolean scoreDocsInOrder, boolean topScorer, Weight weight,
+ DocIdSet docIdSet) throws IOException {
+ final DocIdSetIterator filterIter = docIdSet.iterator();
+ if (filterIter == null) {
+ // this means the filter does not accept any documents.
+ return null;
+ }
+
+ final int firstFilterDoc = filterIter.nextDoc();
+ if (firstFilterDoc == DocIdSetIterator.NO_MORE_DOCS) {
+ return null;
+ }
+ assert firstFilterDoc > -1;
+ // we are gonna advance() this scorer, so we set inorder=true/toplevel=false
+ // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice
+ final Scorer scorer = weight.scorer(context, true, false, null);
+ return (scorer == null) ? null : new LeapFrogScorer(weight, firstFilterDoc, filterIter, scorer);
+ }
+
+ }
+
+ /**
+ * A filter strategy that advances the {@link Scorer} first and consults the
+ * {@link DocIdSet} for each matched document.
+ * + * Use this strategy if the filter computation is more expensive than document + * scoring or if the filter has a linear running time to compute the next + * matching doc like exact geo distances. + *
+ */ + private static final class DocFirstFilterStrategy extends FilterStrategy { + + @Override + public Scorer filteredScorer(final AtomicReaderContext context, + boolean scoreDocsInOrder, boolean topScorer, Weight weight, + DocIdSet docIdSet) throws IOException { + Bits filterAcceptDocs = docIdSet.bits(); + if (filterAcceptDocs == null) { + final DocIdSetIterator filterIter = docIdSet.iterator(); + if (filterIter == null) { + // this means the filter does not accept any documents. + return null; + } + // TODO maybe we should fall back to leap frog + filterAcceptDocs = new Bits() { + @Override + public boolean get(int index) { + try { + return filterIter.docID() == index + || (filterIter.docID() < index && filterIter.advance(index) == index); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public int length() { + return context.reader().maxDoc(); + } + + }; + } + final Scorer scorer = weight.scorer(context, true, false, null); + return (scorer == null) ? null : new DocFirstScorer(weight, + filterAcceptDocs, scorer); + } + + } + } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java index 91e62c8..4d6de37 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestFilteredQuery.java @@ -365,17 +365,16 @@ public class TestFilteredQuery extends LuceneTestCase { } public static final class FilteredQueryRA extends FilteredQuery { - private final boolean useRandomAccess; - public FilteredQueryRA(Query q, Filter f, boolean useRandomAccess) { - super(q,f); - this.useRandomAccess = useRandomAccess; + public FilteredQueryRA(Query q, Filter f, final boolean useRandomAccess) { + super(q, f, new FilteredQuery.RandomAccessFilterStrategy() { + @Override + protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { + return useRandomAccess; + } + }); } - @Override - protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { - return useRandomAccess; - } } } diff --git a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java index db53e4f..f282cf0 100644 --- a/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java +++ b/lucene/spatial/src/java/org/apache/lucene/spatial/vector/TwoDoublesStrategy.java @@ -173,7 +173,7 @@ public class TwoDoublesStrategy extends SpatialStrategy { ValueSourceFilter vsf = new ValueSourceFilter( new QueryWrapperFilter( spatial ), valueSource, 0, circle.getRadius() ); - spatial = new FilteredQuery( new MatchAllDocsQuery(), vsf ); + spatial = new FilteredQuery( new MatchAllDocsQuery(), vsf); } } else if( op == SpatialOperation.IsDisjointTo ) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java index 2f29eea..396358e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.util.Bits; +import org.apache.lucene.util._TestUtil; /** * Helper class that adds some extra checks to ensure correct @@ -101,16 +102,11 @@ public class AssertingIndexSearcher extends IndexSearcher { } }; } - + @Override protected Query wrapFilter(Query query, Filter filter) { if (random.nextBoolean()) return super.wrapFilter(query, filter); - return (filter == null) ? query : new FilteredQuery(query, filter) { - @Override - protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { - return random.nextBoolean(); - } - }; + return (filter == null) ? query : new FilteredQuery(query, filter, _TestUtil.randomFilterStrategy(random)); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java b/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java index 6eb5c39..8516b64 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/search/SearchEquivalenceTestBase.java @@ -173,20 +173,8 @@ public abstract class SearchEquivalenceTestBase extends LuceneTestCase { protected void assertSubsetOf(Query q1, Query q2, Filter filter) throws Exception { // TRUNK ONLY: test both filter code paths if (filter != null && random().nextBoolean()) { - final boolean q1RandomAccess = random().nextBoolean(); - final boolean q2RandomAccess = random().nextBoolean(); - q1 = new FilteredQuery(q1, filter) { - @Override - protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { - return q1RandomAccess; - } - }; - q2 = new FilteredQuery(q2, filter) { - @Override - protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { - return q2RandomAccess; - } - }; + q1 = new FilteredQuery(q1, filter, _TestUtil.randomFilterStrategy(random())); + q2 = new FilteredQuery(q2, filter, _TestUtil.randomFilterStrategy(random())); filter = null; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java index cefdb0f..1de4567 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java @@ -29,7 +29,12 @@ import java.lang.reflect.Method; import java.math.BigDecimal; import java.math.BigInteger; import java.nio.CharBuffer; -import java.util.*; +import java.util.Arrays; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; @@ -72,6 +77,8 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.FilteredQuery; +import org.apache.lucene.search.FilteredQuery.FilterStrategy; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.CompoundFileDirectory; @@ -975,4 +982,27 @@ public class _TestUtil { } } } + + + public static final FilterStrategy randomFilterStrategy(final Random random) { + switch(random.nextInt(5)) { + case 4: + case 3: + new FilteredQuery.RandomAccessFilterStrategy() { + @Override + protected boolean useRandomAccess(Bits bits, int firstFilterDoc) { + return random.nextBoolean(); + } + }; + case 2: + return FilteredQuery.RANDOM_ACCESS_FILTER_STRATEGY; + case 1: + return FilteredQuery.LEAP_FROG_FILTER_STRATEGY; + case 0: + return FilteredQuery.DOC_FIRST_FILTER_STRATEGY; + default: + return FilteredQuery.RANDOM_ACCESS_FILTER_STRATEGY; + } + } + }