Index: src/java/org/apache/lucene/queryParser/QueryParser.jj =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParser.jj (revision 548835) +++ src/java/org/apache/lucene/queryParser/QueryParser.jj (working copy) @@ -638,9 +638,12 @@ protected Query getBooleanQuery(Vector clauses, boolean disableCoord) throws ParseException { + if (clauses.size()==0) { + return null; // all clause words were filtered away by the analyzer. + } BooleanQuery query = new BooleanQuery(disableCoord); for (int i = 0; i < clauses.size(); i++) { - query.add((BooleanClause)clauses.elementAt(i)); + query.add((BooleanClause)clauses.elementAt(i)); } return query; } Index: src/java/org/apache/lucene/queryParser/QueryParser.java =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParser.java (revision 548835) +++ src/java/org/apache/lucene/queryParser/QueryParser.java (working copy) @@ -614,9 +614,12 @@ protected Query getBooleanQuery(Vector clauses, boolean disableCoord) throws ParseException { + if (clauses.size()==0) { + return null; // all clause words were filtered away by the analyzer. + } BooleanQuery query = new BooleanQuery(disableCoord); for (int i = 0; i < clauses.size(); i++) { - query.add((BooleanClause)clauses.elementAt(i)); + query.add((BooleanClause)clauses.elementAt(i)); } return query; } @@ -1258,16 +1261,6 @@ finally { jj_save(0, xla); } } - final private boolean jj_3_1() { - Token xsp; - xsp = jj_scanpos; - if (jj_3R_2()) { - jj_scanpos = xsp; - if (jj_3R_3()) return true; - } - return false; - } - final private boolean jj_3R_3() { if (jj_scan_token(STAR)) return true; if (jj_scan_token(COLON)) return true; @@ -1280,6 +1273,16 @@ return false; } + final private boolean jj_3_1() { + Token xsp; + xsp = jj_scanpos; + if (jj_3R_2()) { + jj_scanpos = xsp; + if (jj_3R_3()) return true; + } + return false; + } + public QueryParserTokenManager token_source; public Token token, jj_nt; private int jj_ntk; Index: src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java =================================================================== --- src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java (revision 548835) +++ src/java/org/apache/lucene/queryParser/MultiFieldQueryParser.java (working copy) @@ -204,13 +204,17 @@ if (queries.length != fields.length) throw new IllegalArgumentException("queries.length != fields.length"); BooleanQuery bQuery = new BooleanQuery(); + int nClauses = 0; for (int i = 0; i < fields.length; i++) { QueryParser qp = new QueryParser(fields[i], analyzer); Query q = qp.parse(queries[i]); - bQuery.add(q, BooleanClause.Occur.SHOULD); + if (q!=null) { + bQuery.add(q, BooleanClause.Occur.SHOULD); + nClauses ++; + } } - return bQuery; + return nClauses>0 ? bQuery : null; } /** @@ -248,12 +252,16 @@ if (fields.length != flags.length) throw new IllegalArgumentException("fields.length != flags.length"); BooleanQuery bQuery = new BooleanQuery(); + int nClauses = 0; for (int i = 0; i < fields.length; i++) { QueryParser qp = new QueryParser(fields[i], analyzer); Query q = qp.parse(query); - bQuery.add(q, flags[i]); + if (q!=null) { + bQuery.add(q, flags[i]); + nClauses ++; + } } - return bQuery; + return nClauses>0 ? bQuery : null; } /** @@ -293,13 +301,17 @@ if (!(queries.length == fields.length && queries.length == flags.length)) throw new IllegalArgumentException("queries, fields, and flags array have have different length"); BooleanQuery bQuery = new BooleanQuery(); + int nClauses = 0; for (int i = 0; i < fields.length; i++) { QueryParser qp = new QueryParser(fields[i], analyzer); Query q = qp.parse(queries[i]); - bQuery.add(q, flags[i]); + if (q!=null) { + bQuery.add(q, flags[i]); + nClauses ++; + } } - return bQuery; + return nClauses>0 ? bQuery : null; } } Index: src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestQueryParser.java (revision 548835) +++ src/test/org/apache/lucene/queryParser/TestQueryParser.java (working copy) @@ -122,6 +122,9 @@ public void assertQueryEquals(String query, Analyzer a, String result) throws Exception { Query q = getQuery(query, a); + if (result==null && q==null) { + return; // expecting null as parse result, e.g. if all words were stopwords. + } String s = q.toString("field"); if (!s.equals(result)) { fail("Query /" + query + "/ yielded /" + s @@ -268,7 +271,7 @@ public void testNumber() throws Exception { // The numbers go away because SimpleAnalzyer ignores them - assertQueryEquals("3", null, ""); + assertQueryEquals("3", null, null); assertQueryEquals("term 1.0 1 2", null, "term"); assertQueryEquals("term term1 term2", null, "term term term"); @@ -365,15 +368,27 @@ } public void testQPA() throws Exception { + assertQueryEquals("term term^3.0 term", qpAnalyzer, "term term^3.0 term"); + assertQueryEquals("term stop^3.0 term", qpAnalyzer, "term term"); + assertQueryEquals("term term term", qpAnalyzer, "term term term"); assertQueryEquals("term +stop term", qpAnalyzer, "term term"); assertQueryEquals("term -stop term", qpAnalyzer, "term term"); + + assertQueryEquals("drop AND (stop) AND roll", qpAnalyzer, "+drop +roll"); + assertQueryEquals("term +(stop) term", qpAnalyzer, "term term"); + assertQueryEquals("term -(stop) term", qpAnalyzer, "term term"); + assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll"); assertQueryEquals("term phrase term", qpAnalyzer, "term \"phrase1 phrase2\" term"); assertQueryEquals("term AND NOT phrase term", qpAnalyzer, "+term -\"phrase1 phrase2\" term"); - assertQueryEquals("stop", qpAnalyzer, ""); + assertQueryEquals("stop^3", qpAnalyzer, null); + assertQueryEquals("stop", qpAnalyzer, null); + assertQueryEquals("(stop)^3", qpAnalyzer, null); + assertQueryEquals("(stop^3)", qpAnalyzer, null); + assertQueryEquals("(stop)", qpAnalyzer, null); assertTrue(getQuery("term term term", qpAnalyzer) instanceof BooleanQuery); assertTrue(getQuery("term +stop", qpAnalyzer) instanceof TermQuery); } @@ -672,7 +687,7 @@ StandardAnalyzer oneStopAnalyzer = new StandardAnalyzer(new String[]{"on"}); QueryParser qp = new QueryParser("field", oneStopAnalyzer); Query q = qp.parse("on^1.0"); - assertNotNull(q); + assertNull(q); q = qp.parse("\"hello\"^2.0"); assertNotNull(q); assertEquals(q.getBoost(), (float) 2.0, (float) 0.5); @@ -680,14 +695,12 @@ assertNotNull(q); assertEquals(q.getBoost(), (float) 2.0, (float) 0.5); q = qp.parse("\"on\"^1.0"); - assertNotNull(q); + assertNull(q); QueryParser qp2 = new QueryParser("field", new StandardAnalyzer()); q = qp2.parse("the^3"); // "the" is a stop word so the result is an empty query: - assertNotNull(q); - assertEquals("", q.toString()); - assertEquals(1.0f, q.getBoost(), 0.01f); + assertNull(q); } public void assertParseException(String queryString) throws Exception { Index: src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (revision 548835) +++ src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (working copy) @@ -42,6 +42,20 @@ */ public class TestMultiFieldQueryParser extends TestCase { + public void testWithStopwords() throws Exception { + String[] fields = {"b", "t"}; + MultiFieldQueryParser mfqp = new MultiFieldQueryParser(fields, new TestQueryParser.QPTestAnalyzer()); + + Query q = mfqp.parse("one"); + assertEquals("b:one t:one", q.toString()); + + q = mfqp.parse("one stop"); + assertEquals("b:one t:one", q.toString()); + + q = mfqp.parse("stop"); + assertEquals(null, q); + } + public void testSimple() throws Exception { String[] fields = {"b", "t"}; MultiFieldQueryParser mfqp = new MultiFieldQueryParser(fields, new StandardAnalyzer()); Index: contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 548835) +++ contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -28,7 +28,7 @@ import java.nio.ByteBuffer; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.Enumeration; +import java.util.Iterator; import java.util.LinkedHashSet; import java.util.Set; @@ -200,8 +200,9 @@ public class MemoryIndexTest extends TestCase { private Analyzer analyzer; - private boolean fastMode = false; + private static final boolean DEBUG = false; + private static final String FIELD_NAME = "content"; /** Runs the tests and/or benchmark */ @@ -223,7 +224,7 @@ "src/java/test/org/apache/lucene/queryParser/*.java", "contrib/memory/src/java/org/apache/lucene/index/memory/*.java", }); - System.out.println("files = " + java.util.Arrays.asList(files)); + dbg("files = " + java.util.Arrays.asList(files)); String[] xargs = new String[] { "1", "1", "memram", "@contrib/memory/src/test/org/apache/lucene/index/memory/testqueries.txt", @@ -265,9 +266,6 @@ } } - boolean toLowerCase = true; -// boolean toLowerCase = false; -// Set stopWords = null; Set stopWords = StopFilter.makeStopSet(StopAnalyzer.ENGLISH_STOP_WORDS); Analyzer[] analyzers = new Analyzer[] { @@ -282,7 +280,7 @@ }; for (int iter=0; iter < iters; iter++) { - System.out.println("\n########### iteration=" + iter); + dbg("\n########### iteration=" + iter); long start = System.currentTimeMillis(); long bytes = 0; @@ -295,11 +293,15 @@ bytes += file.length(); String text = toString(new FileInputStream(file), null); Document doc = createDocument(text); - System.out.println("\n*********** FILE=" + file); + dbg("\n*********** FILE=" + file); for (int q=0; q < queries.length; q++) { try { Query query = parseQuery(queries[q]); + if (query==null) { + dbg("----> skiping null==query("+queries[q]+")"); + continue; + } boolean measureIndexing = false; // toggle this to measure query performance MemoryIndex memind = null; @@ -314,7 +316,7 @@ if (useRAMIndex && measureIndexing) ramind = createRAMIndex(doc); if (useRAMIndex) score2 = query(ramind, query); if (useMemIndex && useRAMIndex) { - System.out.println("diff="+ (score1-score2) + ", query=" + queries[q] + ", s1=" + score1 + ", s2=" + score2); + dbg("diff="+ (score1-score2) + ", query=" + queries[q] + ", s1=" + score1 + ", s2=" + score2); if (score1 != score2 || score1 < 0.0f || score2 < 0.0f || score1 > 1.0f || score2 > 1.0f) { throw new IllegalStateException("BUG DETECTED:" + (i*(q+1)) + " at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); } @@ -323,25 +325,25 @@ } catch (Throwable t) { if (t instanceof OutOfMemoryError) t.printStackTrace(); - System.out.println("Fatal error at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); + dbg("Fatal error at query=" + queries[q] + ", file=" + file + ", anal=" + analyzer); throw t; } } } } long end = System.currentTimeMillis(); - System.out.println("\nsecs = " + ((end-start)/1000.0f)); - System.out.println("queries/sec= " + + dbg("\nsecs = " + ((end-start)/1000.0f)); + dbg("queries/sec= " + (1.0f * runs * queries.length * analyzers.length * files.length / ((end-start)/1000.0f))); float mb = (1.0f * bytes * queries.length * runs) / (1024.0f * 1024.0f); - System.out.println("MB/sec = " + (mb / ((end-start)/1000.0f))); + dbg("MB/sec = " + (mb / ((end-start)/1000.0f))); } if (useMemIndex && useRAMIndex) - System.out.println("No bug found. done."); + dbg("No bug found. done."); else - System.out.println("Done benchmarking (without checking correctness)."); + dbg("Done benchmarking (without checking correctness)."); } // returns file line by line, ignoring empty lines and comments @@ -371,9 +373,9 @@ private MemoryIndex createMemoryIndex(Document doc) { MemoryIndex index = new MemoryIndex(); - Enumeration iter = doc.fields(); - while (iter.hasMoreElements()) { - Field field = (Field) iter.nextElement(); + Iterator iter = doc.getFields().iterator(); + while (iter.hasNext()) { + Field field = (Field) iter.next(); index.addField(field.name(), field.stringValue(), analyzer); } return index; @@ -400,7 +402,7 @@ } private float query(Object index, Query query) { -// System.out.println("MB=" + (getMemorySize(index) / (1024.0f * 1024.0f))); +// dbg("MB=" + (getMemorySize(index) / (1024.0f * 1024.0f))); Searcher searcher = null; try { if (index instanceof Directory) @@ -527,4 +529,10 @@ } } + private void dbg (String s) { + if (DEBUG) { + System.out.println(s); + } + } + }