Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 795767) +++ CHANGES.txt (working copy) @@ -69,8 +69,9 @@ 1. LUCENE-1424: QueryParser now by default uses constant score query rewriting when it generates a WildcardQuery and PrefixQuery (it already does so for RangeQuery, as well). Call - setConstantScoreRewrite(false) to revert to BooleanQuery rewriting - method. (Mark Miller via Mike McCandless) + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY) + to revert to BooleanQuery rewriting method. (Mark Miller via Mike + McCandless) 2. LUCENE-1575: As of 2.9, the core collectors as well as IndexSearcher's search methods that return top N results, no @@ -434,7 +435,7 @@ 6. LUCENE-1424: Moved constant score query rewrite capability into MultiTermQuery, allowing TermRangeQuery, PrefixQuery and WildcardQuery to switch betwen constant-score rewriting or BooleanQuery - expansion rewriting via a new setConstantScoreRewrite method. + expansion rewriting via a new setRewriteMethod method. Deprecated ConstantScoreRangeQuery (Mark Miller via Mike McCandless) @@ -579,6 +580,14 @@ On 32 bit platforms, the address space can be very fragmented, so one big ByteBuffer for the whole file may not fit into address space. (Eks Dev via Uwe Schindler) + +33. LUCENE-1644: Enable 3 rewrite modes for queries deriving from + MultiTermQuery (WildcardQuery, PrefixQuery, TermRangeQuery, + NumericRangeQuery): FILTER first creates a filter and then assigns + constant score (boost) to docs; CONSTANT_BOOLEAN_QUERY create a + BooleanQuery but uses a constant score (boost); + SCORING_BOOLEAN_QUERY also creates a BooleanQuery but keeps the + BooleanQuery's scores. (Mike McCandless) Optimizations Index: src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestQueryParser.java (revision 795767) +++ src/test/org/apache/lucene/queryParser/TestQueryParser.java (working copy) @@ -46,6 +46,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.FuzzyQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; @@ -433,11 +434,11 @@ public void testRange() throws Exception { assertQueryEquals("[ a TO z]", null, "[a TO z]"); - assertTrue(((TermRangeQuery)getQuery("[ a TO z]", null)).getConstantScoreRewrite()); + assertEquals(MultiTermQuery.RewriteMethod.FILTER, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod()); QueryParser qp = new QueryParser("field", new SimpleAnalyzer()); - qp.setConstantScoreRewrite(false); - assertFalse(((TermRangeQuery)qp.parse("[ a TO z]")).getConstantScoreRewrite()); + qp.setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); + assertEquals(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod()); assertQueryEquals("[ a TO z ]", null, "[a TO z]"); assertQueryEquals("{ a TO z}", null, "{a TO z}"); @@ -476,7 +477,7 @@ // supported). // Test ConstantScoreRangeQuery - qp.setConstantScoreRewrite(true); + qp.setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); ScoreDoc[] result = is.search(qp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs; assertEquals("The index Term should not be included.", 0, result.length); @@ -484,7 +485,7 @@ assertEquals("The index Term should be included.", 1, result.length); // Test TermRangeQuery - qp.setConstantScoreRewrite(false); + qp.setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); result = is.search(qp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs; assertEquals("The index Term should not be included.", 0, result.length); Index: src/test/org/apache/lucene/search/TestMultiTermConstantScore.java =================================================================== --- src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (revision 795767) +++ src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (working copy) @@ -88,29 +88,35 @@ /** macro for readability */ public static Query csrq(String f, String l, String h, boolean il, boolean ih) { TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih); - query.setConstantScoreRewrite(true); + query.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); return query; } + public static Query csrq(String f, String l, String h, boolean il, boolean ih, MultiTermQuery.RewriteMethod method) { + TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih); + query.setRewriteMethod(method); + return query; + } + /** macro for readability */ public static Query csrq(String f, String l, String h, boolean il, boolean ih, Collator c) { TermRangeQuery query = new TermRangeQuery(f, l, h, il, ih, c); - query.setConstantScoreRewrite(true); + query.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); return query; } /** macro for readability */ public static Query cspq(Term prefix) { PrefixQuery query = new PrefixQuery(prefix); - query.setConstantScoreRewrite(true); + query.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); return query; } /** macro for readability */ public static Query cswcq(Term wild) { WildcardQuery query = new WildcardQuery(wild); - query.setConstantScoreRewrite(true); + query.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); return query; } @@ -156,6 +162,14 @@ result[i].score); } + result = search.search(csrq("data", "1", "6", T, T, MultiTermQuery.RewriteMethod.CONSTANT_BOOLEAN_QUERY), null, 1000).scoreDocs; + numHits = result.length; + assertEquals("wrong number of results", 6, numHits); + for (int i = 0; i < numHits; i++) { + assertEquals("score for " + i + " was not the same", score, + result[i].score); + } + } public void testBoost() throws IOException { @@ -201,6 +215,18 @@ assertEquals(0, hits[1].doc); assertTrue(hits[0].score > hits[1].score); + q1 = csrq("data", "A", "A", T, T, MultiTermQuery.RewriteMethod.CONSTANT_BOOLEAN_QUERY); // matches document #0 + q1.setBoost(.1f); + q2 = csrq("data", "Z", "Z", T, T, MultiTermQuery.RewriteMethod.CONSTANT_BOOLEAN_QUERY); // matches document #1 + bq = new BooleanQuery(true); + bq.add(q1, BooleanClause.Occur.SHOULD); + bq.add(q2, BooleanClause.Occur.SHOULD); + + hits = search.search(bq, null, 1000).scoreDocs; + assertEquals(1, hits[0].doc); + assertEquals(0, hits[1].doc); + assertTrue(hits[0].score > hits[1].score); + q1 = csrq("data", "A", "A", T, T); // matches document #0 q1.setBoost(10f); q2 = csrq("data", "Z", "Z", T, T); // matches document #1 Index: src/test/org/apache/lucene/search/TestNumericRangeQuery32.java =================================================================== --- src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (revision 795767) +++ src/test/org/apache/lucene/search/TestNumericRangeQuery32.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.document.NumericField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter.MaxFieldLength; -import org.apache.lucene.search.SortField; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; @@ -104,13 +103,13 @@ switch (i) { case 0: type = " (constant score)"; - q.setConstantScoreRewrite(true); + q.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER); terms = q.getTotalNumberOfTerms(); break; case 1: type = " (boolean query)"; - q.setConstantScoreRewrite(false); + q.setRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER); terms = q.getTotalNumberOfTerms(); break; @@ -229,7 +228,7 @@ // test inclusive range NumericRangeQuery tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, true); TermRangeQuery cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, true); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); TopDocs tTopDocs = searcher.search(tq, 1); TopDocs cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -238,7 +237,7 @@ // test exclusive range tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, false); cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, false); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -247,7 +246,7 @@ // test left exclusive range tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), false, true); cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), false, true); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -256,7 +255,7 @@ // test right exclusive range tq=NumericRangeQuery.newIntRange(field, precisionStep, new Integer(lower), new Integer(upper), true, false); cq=new TermRangeQuery(field, NumericUtils.intToPrefixCoded(lower), NumericUtils.intToPrefixCoded(upper), true, false); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); Index: src/test/org/apache/lucene/search/TestNumericRangeQuery64.java =================================================================== --- src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (revision 795767) +++ src/test/org/apache/lucene/search/TestNumericRangeQuery64.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.document.NumericField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriter.MaxFieldLength; -import org.apache.lucene.search.SortField; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.NumericUtils; @@ -108,13 +107,13 @@ switch (i) { case 0: type = " (constant score)"; - q.setConstantScoreRewrite(true); + q.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER); terms = q.getTotalNumberOfTerms(); break; case 1: type = " (boolean query)"; - q.setConstantScoreRewrite(false); + q.setRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); topDocs = searcher.search(q, null, noDocs, Sort.INDEXORDER); terms = q.getTotalNumberOfTerms(); break; @@ -245,7 +244,7 @@ // test inclusive range NumericRangeQuery tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, true); TermRangeQuery cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, true); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); TopDocs tTopDocs = searcher.search(tq, 1); TopDocs cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -254,7 +253,7 @@ // test exclusive range tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, false); cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, false); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -263,7 +262,7 @@ // test left exclusive range tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), false, true); cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), false, true); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); @@ -272,7 +271,7 @@ // test right exclusive range tq=NumericRangeQuery.newLongRange(field, precisionStep, new Long(lower), new Long(upper), true, false); cq=new TermRangeQuery(field, NumericUtils.longToPrefixCoded(lower), NumericUtils.longToPrefixCoded(upper), true, false); - cq.setConstantScoreRewrite(true); + cq.setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); tTopDocs = searcher.search(tq, 1); cTopDocs = searcher.search(cq, 1); assertEquals("Returned count for NumericRangeQuery and TermRangeQuery must be equal", cTopDocs.totalHits, tTopDocs.totalHits ); Index: src/java/org/apache/lucene/queryParser/Token.java =================================================================== --- src/java/org/apache/lucene/queryParser/Token.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/Token.java (working copy) @@ -121,4 +121,4 @@ } } -/* JavaCC - OriginalChecksum=c147cc166a7cf8812c7c39bc8c5eb868 (do not edit this line) */ +/* JavaCC - OriginalChecksum=37b1923f964a5a434f5ea3d6952ff200 (do not edit this line) */ Index: src/java/org/apache/lucene/queryParser/TokenMgrError.java =================================================================== --- src/java/org/apache/lucene/queryParser/TokenMgrError.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/TokenMgrError.java (working copy) @@ -137,4 +137,4 @@ this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason); } } -/* JavaCC - OriginalChecksum=186d5bcc64733844c7daab5ad5a6e349 (do not edit this line) */ +/* JavaCC - OriginalChecksum=55cddb2336a66b376c0bb59d916b326d (do not edit this line) */ Index: src/java/org/apache/lucene/queryParser/QueryParser.java =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParser.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/QueryParser.java (working copy) @@ -25,6 +25,8 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; // for javadocs import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; @@ -118,7 +120,7 @@ private Operator operator = OR_OPERATOR; boolean lowercaseExpandedTerms = true; - boolean constantScoreRewrite= true; + MultiTermQuery.RewriteMethod multiTermRewriteMethod = MultiTermQuery.RewriteMethod.FILTER; boolean allowLeadingWildcard = false; boolean enablePositionIncrements = false; @@ -331,40 +333,48 @@ } /** - * @deprecated Please use {@link #setConstantScoreRewrite} instead. + * @deprecated Please use {@link #setMultiTermRewriteMethod} instead. */ public void setUseOldRangeQuery(boolean useOldRangeQuery) { - constantScoreRewrite = !useOldRangeQuery; + if (useOldRangeQuery) { + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); + } else { + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); + } } /** - * @deprecated Please use {@link #getConstantScoreRewrite} instead. + * @deprecated Please use {@link #getMultiTermRewriteMethod} instead. */ public boolean getUseOldRangeQuery() { - return !constantScoreRewrite; + if (getMultiTermRewriteMethod() == MultiTermQuery.RewriteMethod.FILTER) { + return false; + } else { + return true; + } } /** - * By default QueryParser uses constant-score rewriting + * By default QueryParser uses {@link RewriteMethod#FILTER} rewriting * when creating a PrefixQuery, WildcardQuery or RangeQuery. This implementation is generally preferable because it * a) Runs faster b) Does not have the scarcity of terms unduly influence score * c) avoids any "TooManyBooleanClauses" exception. * However, if your application really needs to use the * old-fashioned BooleanQuery expansion rewriting and the above - * points are not relevant then set this option to true - * Default is false. + * points are not relevant then use this to change + * the rewrite method. */ - public void setConstantScoreRewrite(boolean v) { - constantScoreRewrite = v; + public void setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod method) { + multiTermRewriteMethod = method; } /** - * @see #setConstantScoreRewrite(boolean) + * @see #setMultiTermRewriteMethod */ - public boolean getConstantScoreRewrite() { - return constantScoreRewrite; + public MultiTermQuery.RewriteMethod getMultiTermRewriteMethod() { + return multiTermRewriteMethod; } /** @@ -858,7 +868,7 @@ */ protected Query newPrefixQuery(Term prefix){ PrefixQuery query = new PrefixQuery(prefix); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } @@ -884,7 +894,7 @@ */ protected Query newRangeQuery(String field, String part1, String part2, boolean inclusive) { final TermRangeQuery query = new TermRangeQuery(field, part1, part2, inclusive, inclusive, rangeCollator); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } @@ -903,7 +913,7 @@ */ protected Query newWildcardQuery(Term t) { WildcardQuery query = new WildcardQuery(t); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } Index: src/java/org/apache/lucene/queryParser/QueryParser.jj =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParser.jj (revision 795767) +++ src/java/org/apache/lucene/queryParser/QueryParser.jj (working copy) @@ -49,6 +49,8 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; // for javadocs import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; @@ -142,7 +144,7 @@ private Operator operator = OR_OPERATOR; boolean lowercaseExpandedTerms = true; - boolean constantScoreRewrite= true; + MultiTermQuery.RewriteMethod multiTermRewriteMethod = MultiTermQuery.RewriteMethod.FILTER; boolean allowLeadingWildcard = false; boolean enablePositionIncrements = false; @@ -355,40 +357,48 @@ } /** - * @deprecated Please use {@link #setConstantScoreRewrite} instead. + * @deprecated Please use {@link #setMultiTermRewriteMethod} instead. */ public void setUseOldRangeQuery(boolean useOldRangeQuery) { - constantScoreRewrite = !useOldRangeQuery; + if (useOldRangeQuery) { + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); + } else { + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); + } } /** - * @deprecated Please use {@link #getConstantScoreRewrite} instead. + * @deprecated Please use {@link #getMultiTermRewriteMethod} instead. */ public boolean getUseOldRangeQuery() { - return !constantScoreRewrite; + if (getMultiTermRewriteMethod() == MultiTermQuery.RewriteMethod.FILTER) { + return false; + } else { + return true; + } } /** - * By default QueryParser uses constant-score rewriting + * By default QueryParser uses {@link MultiTermQuery.RewriteMethod#FILTER} rewriting * when creating a PrefixQuery, WildcardQuery or RangeQuery. This implementation is generally preferable because it * a) Runs faster b) Does not have the scarcity of terms unduly influence score * c) avoids any "TooManyBooleanClauses" exception. * However, if your application really needs to use the * old-fashioned BooleanQuery expansion rewriting and the above - * points are not relevant then set this option to true - * Default is false. + * points are not relevant then use this to change + * the rewrite method. */ - public void setConstantScoreRewrite(boolean v) { - constantScoreRewrite = v; + public void setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod method) { + multiTermRewriteMethod = method; } /** - * @see #setConstantScoreRewrite(boolean) + * @see #setMultiTermRewriteMethod */ - public boolean getConstantScoreRewrite() { - return constantScoreRewrite; + public MultiTermQuery.RewriteMethod getMultiTermRewriteMethod() { + return multiTermRewriteMethod; } /** @@ -882,7 +892,7 @@ */ protected Query newPrefixQuery(Term prefix){ PrefixQuery query = new PrefixQuery(prefix); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } @@ -908,7 +918,7 @@ */ protected Query newRangeQuery(String field, String part1, String part2, boolean inclusive) { final TermRangeQuery query = new TermRangeQuery(field, part1, part2, inclusive, inclusive, rangeCollator); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } @@ -927,7 +937,7 @@ */ protected Query newWildcardQuery(Term t) { WildcardQuery query = new WildcardQuery(t); - query.setConstantScoreRewrite(constantScoreRewrite); + query.setRewriteMethod(multiTermRewriteMethod); return query; } Index: src/java/org/apache/lucene/queryParser/CharStream.java =================================================================== --- src/java/org/apache/lucene/queryParser/CharStream.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/CharStream.java (working copy) @@ -109,4 +109,4 @@ void Done(); } -/* JavaCC - OriginalChecksum=32a89423891f765dde472f7ef0e3ef7b (do not edit this line) */ +/* JavaCC - OriginalChecksum=a83909a2403f969f94d18375f9f143e4 (do not edit this line) */ Index: src/java/org/apache/lucene/queryParser/ParseException.java =================================================================== --- src/java/org/apache/lucene/queryParser/ParseException.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/ParseException.java (working copy) @@ -195,4 +195,4 @@ } } -/* JavaCC - OriginalChecksum=c7631a240f7446940695eac31d9483ca (do not edit this line) */ +/* JavaCC - OriginalChecksum=c63b396885c4ff44d7aa48d3feae60cd (do not edit this line) */ Index: src/java/org/apache/lucene/queryParser/ComplexPhraseQueryParser.java =================================================================== --- src/java/org/apache/lucene/queryParser/ComplexPhraseQueryParser.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/ComplexPhraseQueryParser.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanNotQuery; @@ -77,9 +78,9 @@ public Query parse(String query) throws ParseException { if (isPass2ResolvingPhrases) { - boolean oldConstantScoreRewriteSetting = getConstantScoreRewrite(); + MultiTermQuery.RewriteMethod oldMethod = getMultiTermRewriteMethod(); try { - // Temporarily set constantScoreRewrite to false so that Parser will + // Temporarily force BooleanQuery rewrite so that Parser will // generate visible // collection of terms which we can convert into SpanQueries. // ConstantScoreRewrite mode produces an @@ -88,10 +89,10 @@ // QueryParser is not guaranteed threadsafe anyway so this temporary // state change should not // present an issue - setConstantScoreRewrite(false); + setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); return super.parse(query); } finally { - setConstantScoreRewrite(oldConstantScoreRewriteSetting); + setMultiTermRewriteMethod(oldMethod); } } @@ -165,7 +166,7 @@ // that can be turned into SpanOr clause TermRangeQuery rangeQuery = new TermRangeQuery(field, part1, part2, inclusive, inclusive, getRangeCollator()); - rangeQuery.setConstantScoreRewrite(false);; + rangeQuery.setRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); return rangeQuery; } return super.newRangeQuery(field, part1, part2, inclusive); Index: src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (revision 795767) +++ src/java/org/apache/lucene/queryParser/QueryParserTokenManager.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; import org.apache.lucene.search.PhraseQuery; Index: src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/MultiTermQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -18,10 +18,12 @@ */ import java.io.IOException; +import java.io.Serializable; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.util.Parameter; import org.apache.lucene.queryParser.QueryParser; // for javadoc /** @@ -34,24 +36,64 @@ * FilteredTermEnum} that iterates through the terms to be * matched. * - *

NOTE: if {@link #setConstantScoreRewrite} is - * false, you may encounter a {@link + *

NOTE: if {@link #setRewriteMethod} is either + * {@link RewriteMethod#CONSTANT_BOOLEAN_QUERY} or + * {@link RewriteMethod#SCORING_BOOLEAN_QUERY}, + * you may encounter a {@link * BooleanQuery.TooManyClauses} exception during searching, * which happens when the number of terms to be searched * exceeds {@link BooleanQuery#getMaxClauseCount()}. - * Setting {@link #setConstantScoreRewrite} to false + * Setting {@link #setRewriteMethod} to {@link RewriteMethod#FILTER} * prevents this. * * Note that {@link QueryParser} by default produces - * MultiTermQueries with {@link #setConstantScoreRewrite} - * true. + * MultiTermQueries with {@link #setRewriteMethod} + * {@link RewriteMethod#FILTER}. */ public abstract class MultiTermQuery extends Query { /* @deprecated move to sub class */ protected Term term; - protected boolean constantScoreRewrite = false; + protected RewriteMethod rewriteMethod = RewriteMethod.SCORING_BOOLEAN_QUERY; transient int numberOfTerms = 0; + public static final class RewriteMethod extends Parameter implements Serializable { + private RewriteMethod(String name) { + super(name); + } + + /** The FILTER rewrite method first creates a private + * Filter, by visiting each term in sequence and + * marking all docs for that term. Matching documents + * are assigned a constant score equal to the query's + * boost. This method is often faster than the + * BooleanQuery methods, and it will never hit an + * errant {@link BooleanQuery.TooManyClauses} + * exception. */ + public static final RewriteMethod FILTER = new RewriteMethod("FILTER"); + + /** The CONSTANT_BOOLEAN_QUERY rewrite method translates + * each term into {@link BooleanClause.Occur#SHOULD} + * clause in a BooleanQuery, but turns off the normal + * scoring done by BooleanQuery. Instead, each + * matching document receives a constant score equal to + * the query's boost. This rewite method can hit + * {@link BooleanQuery.TooManyClauses} if the number of + * terms exceeds {@link + * BooleanQuery#getMaxClauseCount}. */ + public static final RewriteMethod CONSTANT_BOOLEAN_QUERY = new RewriteMethod("CONSTANT_BOOLEAN_QUERY"); + + /** The SCORING_BOOLEAN_QUERY rewrite method translates + * each term into {@link BooleanClause.Occur#SHOULD} + * clause in a BooleanQuery, and keeps the scores as + * computed by the query. Note that typically such + * scores have little meaning, so it's better to use + * {@link #FILTER} or {@link #CONSTANT_BOOLEAN_QUERY} + * instead. This rewite method can hit {@link + * BooleanQuery.TooManyClauses} if the number of terms + * exceeds {@link BooleanQuery#getMaxClauseCount}. */ + public static final RewriteMethod SCORING_BOOLEAN_QUERY = new RewriteMethod("SCORING_BOOLEAN_QUERY"); + } + /** Constructs a query for terms matching term. */ public MultiTermQuery(Term term) { this.term = term; @@ -110,7 +152,7 @@ } public Query rewrite(IndexReader reader) throws IOException { - if (!constantScoreRewrite) { + if (rewriteMethod == RewriteMethod.CONSTANT_BOOLEAN_QUERY || rewriteMethod == RewriteMethod.SCORING_BOOLEAN_QUERY) { FilteredTermEnum enumerator = getEnum(reader); BooleanQuery query = new BooleanQuery(true); try { @@ -126,11 +168,21 @@ } finally { enumerator.close(); } - return query; - } else { + + Query result = query; + + if (rewriteMethod == RewriteMethod.CONSTANT_BOOLEAN_QUERY) { + // strips the scores off + result = new ConstantScoreQuery(new QueryWrapperFilter(result)); + result.setBoost(getBoost()); + } + return result; + } else if (rewriteMethod == RewriteMethod.FILTER) { Query query = new ConstantScoreQuery(getFilter()); query.setBoost(getBoost()); return query; + } else { + throw new IllegalStateException("unknown rewrite method " + rewriteMethod); } } @@ -155,10 +207,10 @@ } /** - * @see #setConstantScoreRewrite + * @see #setRewriteMethod */ - public boolean getConstantScoreRewrite() { - return constantScoreRewrite; + public RewriteMethod getRewriteMethod() { + return rewriteMethod; } /** @@ -181,8 +233,8 @@ * query's boost. * */ - public void setConstantScoreRewrite(boolean constantScoreRewrite) { - this.constantScoreRewrite = constantScoreRewrite; + public void setRewriteMethod(RewriteMethod method) { + rewriteMethod = method; } //@Override @@ -190,7 +242,16 @@ final int prime = 31; int result = 1; result = prime * result + Float.floatToIntBits(getBoost()); - result = prime * result + (constantScoreRewrite ? 1231 : 1237); + result = prime * result; + if (rewriteMethod == RewriteMethod.FILTER) { + result += 1231; + } else if (rewriteMethod == RewriteMethod.SCORING_BOOLEAN_QUERY) { + result += 1237; + } else if (rewriteMethod == RewriteMethod.CONSTANT_BOOLEAN_QUERY) { + result += 1249; + } else { + throw new IllegalStateException("unknown rewrite method " + rewriteMethod); + } return result; } @@ -205,7 +266,7 @@ MultiTermQuery other = (MultiTermQuery) obj; if (Float.floatToIntBits(getBoost()) != Float.floatToIntBits(other.getBoost())) return false; - if (constantScoreRewrite != other.constantScoreRewrite) + if (rewriteMethod != other.rewriteMethod) return false; return true; } Index: src/java/org/apache/lucene/search/FuzzyQuery.java =================================================================== --- src/java/org/apache/lucene/search/FuzzyQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/FuzzyQuery.java (working copy) @@ -71,6 +71,7 @@ this.minimumSimilarity = minimumSimilarity; this.prefixLength = prefixLength; + rewriteMethod = RewriteMethod.SCORING_BOOLEAN_QUERY; } /** @@ -115,8 +116,8 @@ return term; } - public void setConstantScoreRewrite(boolean constantScoreRewrite) { - throw new UnsupportedOperationException("FuzzyQuery cannot rewrite to a constant score query"); + public void setRewriteMethod(RewriteMethod method) { + throw new UnsupportedOperationException("FuzzyQuery cannot change rewrite method"); } public Query rewrite(IndexReader reader) throws IOException { Index: src/java/org/apache/lucene/search/RangeQuery.java =================================================================== --- src/java/org/apache/lucene/search/RangeQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/RangeQuery.java (working copy) @@ -30,8 +30,7 @@ * supplied range according to {@link Term#compareTo(Term)}. It is not intended * for numerical ranges, use {@link NumericRangeQuery} instead. * - *

This query is in - * {@linkplain MultiTermQuery#setConstantScoreRewrite(boolean) boolean query rewrite mode}. + *

This query is in {@linkplain MultiTermQuery#setRewriteMethod filter rewrite mode}. * If you want to change this, use the new {@link TermRangeQuery} instead. * * @deprecated Use {@link TermRangeQuery} for term ranges or @@ -93,7 +92,7 @@ inclusive, inclusive, collator ); - delegate.setConstantScoreRewrite(false); + delegate.setRewriteMethod(TermRangeQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); } public void setBoost(float b) { Index: src/java/org/apache/lucene/search/PrefixQuery.java =================================================================== --- src/java/org/apache/lucene/search/PrefixQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/PrefixQuery.java (working copy) @@ -24,7 +24,14 @@ import org.apache.lucene.util.ToStringUtils; /** A Query that matches documents containing terms with a specified prefix. A PrefixQuery - * is built by QueryParser for input like app*. */ + * is built by QueryParser for input like app*. + * + *

NOTE: Currently this query uses {@link + * MultiTermQuery.RewriteMethod#SCORING_BOOLEAN_QUERY}, + * which assigns not-very-useful scores to the resulting + * hits. In 3.0 this default will change to {@link + * MultiTermQuery.RewriteMethod#FILTER}; you can use {@link + * MultiTermQuery#setRewriteMethod} to change it. */ public class PrefixQuery extends MultiTermQuery { private Term prefix; Index: src/java/org/apache/lucene/search/ConstantScoreRangeQuery.java =================================================================== --- src/java/org/apache/lucene/search/ConstantScoreRangeQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/ConstantScoreRangeQuery.java (working copy) @@ -29,9 +29,8 @@ * supplied range according to {@link String#compareTo(String)}. It is not intended * for numerical ranges, use {@link NumericRangeQuery} instead. * - *

This query is in - * {@linkplain MultiTermQuery#setConstantScoreRewrite(boolean) constant score rewrite mode}. - * If you want to change this, use the new {@link TermRangeQuery} instead. + *

This query is hardwired to {@link MultiTermQuery.RewriteMethod#FILTER}. + * If you want to change this, use {@link TermRangeQuery} instead. * * @deprecated Use {@link TermRangeQuery} for term ranges or * {@link NumericRangeQuery} for numeric ranges instead. @@ -44,14 +43,14 @@ public ConstantScoreRangeQuery(String fieldName, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper) { super(fieldName, lowerVal, upperVal, includeLower, includeUpper); - this.constantScoreRewrite = true; + rewriteMethod = RewriteMethod.FILTER; } public ConstantScoreRangeQuery(String fieldName, String lowerVal, String upperVal, boolean includeLower, boolean includeUpper, Collator collator) { super(fieldName, lowerVal, upperVal, includeLower, includeUpper, collator); - this.constantScoreRewrite = true; + rewriteMethod = RewriteMethod.FILTER; } public String getLowerVal() { @@ -63,8 +62,9 @@ } /** Changes of mode are not supported by this class (fixed to constant score rewrite mode) */ - public void setConstantScoreRewrite(boolean constantScoreRewrite) { - if (!constantScoreRewrite) - throw new UnsupportedOperationException("Use TermRangeQuery instead to enable boolean query rewrite."); + public void setRewriteMethod(RewriteMethod method) { + if (method != RewriteMethod.FILTER) { + throw new UnsupportedOperationException("Use TermRangeQuery instead to enable scoring or constant boolean query rewrite."); + } } } Index: src/java/org/apache/lucene/search/NumericRangeQuery.java =================================================================== --- src/java/org/apache/lucene/search/NumericRangeQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/NumericRangeQuery.java (working copy) @@ -123,12 +123,12 @@ * *

Comparisions of the different types of RangeQueries on an index with about 500,000 docs showed * that {@link TermRangeQuery} in boolean rewrite mode (with raised {@link BooleanQuery} clause count) - * took about 30-40 secs to complete, {@link TermRangeQuery} in constant score rewrite mode took 5 secs + * took about 30-40 secs to complete, {@link TermRangeQuery} in filter rewrite mode took 5 secs * and executing this class took <100ms to complete (on an Opteron64 machine, Java 1.5, 8 bit * precision step). This query type was developed for a geographic portal, where the performance for * e.g. bounding boxes or exact date/time stamps is important.

* - *

The query defaults to {@linkplain #setConstantScoreRewrite constant score rewrite mode}. + *

The query defaults to {@linkplain MultiTermQuery.RewriteMethod#FILTER} * With precision steps of ≤4, this query can be run in conventional {@link BooleanQuery} * rewrite mode without changing the max clause count. * @@ -152,7 +152,7 @@ this.max = max; this.minInclusive = minInclusive; this.maxInclusive = maxInclusive; - setConstantScoreRewrite(true); + setRewriteMethod(RewriteMethod.FILTER); } /** Index: src/java/org/apache/lucene/search/TermRangeQuery.java =================================================================== --- src/java/org/apache/lucene/search/TermRangeQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/TermRangeQuery.java (working copy) @@ -20,7 +20,6 @@ import java.io.IOException; import java.text.Collator; -import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.ToStringUtils; @@ -31,9 +30,9 @@ * supplied range according to {@link String#compareTo(String)}. It is not intended * for numerical ranges, use {@link NumericRangeQuery} instead. * - *

This query is in constant score mode per default. - * See {@link MultiTermQuery#setConstantScoreRewrite} for the tradeoffs between - * enabling and disabling constantScoreRewrite mode. + *

This query is in filter rewrite mode per default. + * See {@link MultiTermQuery#setRewriteMethod} for the + * tradeoffs of different rewrite methods. * @since 2.9 */ @@ -110,7 +109,7 @@ this.includeLower = includeLower; this.includeUpper = includeUpper; this.collator = collator; - this.constantScoreRewrite = true; + setRewriteMethod(RewriteMethod.FILTER); } /** Returns the field name for this query */ Index: src/java/org/apache/lucene/search/WildcardQuery.java =================================================================== --- src/java/org/apache/lucene/search/WildcardQuery.java (revision 795767) +++ src/java/org/apache/lucene/search/WildcardQuery.java (working copy) @@ -30,8 +30,14 @@ * a Wildcard term should not start with one of the wildcards * or * ?. * - * @see WildcardTermEnum - */ + *

NOTE: Currently this query uses {@link + * MultiTermQuery.RewriteMethod#SCORING_BOOLEAN_QUERY}, + * which assigns not-very-useful scores to the resulting + * hits. In 3.0 this default will change to {@link + * MultiTermQuery.RewriteMethod#FILTER}; you can use {@link + * MultiTermQuery#setRewriteMethod} to change it. + * + * @see WildcardTermEnum */ public class WildcardQuery extends MultiTermQuery { private boolean termContainsWildcard; protected Term term; Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java =================================================================== --- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 795767) +++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy) @@ -53,6 +53,7 @@ import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.ConstantScoreRangeQuery; import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.Hits; @@ -548,7 +549,7 @@ numHighlights = 0; query = new WildcardQuery(new Term(FIELD_NAME, "ken*")); - ((WildcardQuery)query).setConstantScoreRewrite(true); + ((WildcardQuery)query).setRewriteMethod(MultiTermQuery.RewriteMethod.FILTER); searcher = new IndexSearcher(ramDir); // can't rewrite ConstantScore if you want to highlight it - // it rewrites to ConstantScoreQuery which cannot be highlighted @@ -1186,7 +1187,7 @@ searchers[1] = new IndexSearcher(ramDir2); MultiSearcher multiSearcher = new MultiSearcher(searchers); QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer()); - parser.setConstantScoreRewrite(false); + parser.setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); query = parser.parse("multi*"); System.out.println("Searching for: " + query.toString(FIELD_NAME)); // at this point the multisearcher calls combine(query[]) @@ -1487,7 +1488,7 @@ public void doSearching(String queryString) throws Exception { QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer()); - parser.setConstantScoreRewrite(false); + parser.setMultiTermRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); query = parser.parse(queryString); doSearching(query); } Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (revision 795767) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (working copy) @@ -28,14 +28,11 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.index.FilterIndexReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermEnum; import org.apache.lucene.index.memory.MemoryIndex; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.ConstantScoreRangeQuery; import org.apache.lucene.search.DisjunctionMaxQuery; import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.FuzzyQuery; @@ -139,9 +136,9 @@ terms.putAll(disjunctTerms); } else if (query instanceof MultiTermQuery && (highlightCnstScrRngQuery || expandMultiTermQuery)) { MultiTermQuery mtq = ((MultiTermQuery)query); - if(mtq.getConstantScoreRewrite()) { + if(mtq.getRewriteMethod() == MultiTermQuery.RewriteMethod.FILTER) { mtq = copyMultiTermQuery(mtq); - mtq.setConstantScoreRewrite(false); + mtq.setRewriteMethod(MultiTermQuery.RewriteMethod.SCORING_BOOLEAN_QUERY); query = mtq; } String field;