Index: src/java/org/apache/lucene/queryParser/QueryParserExtension.java =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParserExtension.java (revision 0) +++ src/java/org/apache/lucene/queryParser/QueryParserExtension.java (revision 0) @@ -0,0 +1,99 @@ +package org.apache.lucene.queryParser; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; + +/** + * This class represents an extension base class to the Lucene standard + * {@link QueryParser}. The {@link QueryParser} is generated by the JavaCC + * parser generator. Changing or adding functionality or syntax in the standard + * query parser requires changes to the JavaCC source file. To enable extending + * the standard query parser without changing the JavaCC sources and re-generate + * the parser the {@link QueryParserExtension} can be customized. + */ +public abstract class QueryParserExtension { + + /** + * Parses an embedded query string, returning a + * {@link org.apache.lucene.search.Query}. + * + * @param embeddedQuery + * the embedded query string to parse + * @param field + * the field specified for the embedded query string + * @return a {@link Query} instance representing the parsed embedded query + * string. + * @throws ParseException + * if the parsing fails + */ + public abstract Query parse(final String embeddedQuery, final String field) + throws ParseException; + + /** + * Returns a String where those characters that QueryParserExtension expects + * to be escaped are escaped by a preceding \. + * + * @param queryString + * the query string to escape + * @return a String where those characters that QueryParserExtension expects + * to be escaped are escaped by a preceding \. + */ + public static String escape(final String queryString) { + final StringBuilder sb = new StringBuilder(); + for (int i = 0; i < queryString.length(); i++) { + char c = queryString.charAt(i); + if (c == '\\' || c == '/') { + sb.append('\\'); + } + sb.append(c); + } + return sb.toString(); + } + + static String discardEscapeChars(final String input) { + final char[] output = new char[input.length()]; + int length = 0; + boolean seenEscapeChar = false; + for (int i = 0; i < output.length; i++) { + char currentChar = input.charAt(i); + if (seenEscapeChar) { + switch (currentChar) { + case '\\': + case '/': + output[length++] = currentChar; + break; + default: + // last char has been omitted + output[length++] = '\\'; + output[length++] = currentChar; + } + seenEscapeChar = false; + } else if (currentChar == '\\') { + seenEscapeChar = true; + } else { + output[length++] = currentChar; + } + } + + return new String(output, 0, length); + } + + /** + * + * A default implementation of {@link QueryParserExtension} that builds a + * {@link TermQuery} instance from the embedded query string. + * + */ + static class DefaultParserExtension extends QueryParserExtension { + + static QueryParserExtension DEFAULT = new DefaultParserExtension(); + + @Override + public Query parse(String embeddedQuery, String field) + throws ParseException { + return new TermQuery(new Term(field, embeddedQuery)); + } + + } +} Index: src/java/org/apache/lucene/queryParser/QueryParser.jj =================================================================== --- src/java/org/apache/lucene/queryParser/QueryParser.jj (revision 833478) +++ src/java/org/apache/lucene/queryParser/QueryParser.jj (working copy) @@ -45,6 +45,7 @@ import org.apache.lucene.document.DateField; import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.QueryParserExtension.DefaultParserExtension; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; @@ -118,7 +119,17 @@ * {@link #getRangeQuery(String, String, String, boolean)} to * use a different method for date conversion. *

- * + *

+ * Although this class has been generated by the JavaCC parser generator it + * provides rudimentary support for extending the generated parser. A sequence + * of characters enclosed by a leading and trailing forward slash ('/') are not + * interpreted as a query string but passed to an instance of + * {@link QueryParserExtension} to build a corresponding {@link Query} instance. + * The parsing of such an embedded query string is left to the extension. + * Within an extension query string only forward- and backslash characters + * are treated as a special char and need to be escaped. See + * {@link QueryParserExtension#escape(String)} for details. + *

*

Note that QueryParser is not thread-safe.

* *

NOTE: there is a new QueryParser in contrib, which matches @@ -164,6 +175,7 @@ float fuzzyMinSim = FuzzyQuery.defaultMinSimilarity; int fuzzyPrefixLength = FuzzyQuery.defaultPrefixLength; Locale locale = Locale.getDefault(); + QueryParserExtension extension = DefaultParserExtension.DEFAULT; // the default date resolution DateTools.Resolution dateResolution = null; @@ -224,7 +236,25 @@ } } - /** + /** + * Sets the {@link QueryParserExtension} to use + * @param extension the {@link QueryParserExtension} to use + */ + public void setExtension(QueryParserExtension extension) + { + this.extension = extension; + } + + /** + * Returns the currently in use {@link QueryParserExtension} instance. + * @return the currently in use {@link QueryParserExtension} instance. + */ + public QueryParserExtension getExtension() + { + return this.extension; + } + + /** * @return Returns the analyzer. */ public Analyzer getAnalyzer() { @@ -911,7 +941,7 @@ return null; // all clause words were filtered away by the analyzer. } BooleanQuery query = newBooleanQuery(disableCoord); - for(final BooleanClause clause: clauses) { + for(BooleanClause clause: clauses) { query.add(clause); } return query; @@ -1098,7 +1128,7 @@ // These characters are part of the query syntax and must be escaped if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')' || c == ':' || c == '^' || c == '[' || c == ']' || c == '\"' || c == '{' || c == '}' || c == '~' - || c == '*' || c == '?' || c == '|' || c == '&') { + || c == '*' || c == '?' || c == '|' || c == '&' || c == '/') { sb.append('\\'); } sb.append(c); @@ -1134,11 +1164,13 @@ // every character that follows a backslash is considered as an escaped character | <#_ESCAPED_CHAR: "\\" ~[] > | <#_TERM_START_CHAR: ( ~[ " ", "\t", "\n", "\r", "\u3000", "+", "-", "!", "(", ")", ":", "^", - "[", "]", "\"", "{", "}", "~", "*", "?", "\\" ] + "[", "]", "\"", "{", "}", "~", "*", "?", "\\", "/" ] | <_ESCAPED_CHAR> ) > | <#_TERM_CHAR: ( <_TERM_START_CHAR> | <_ESCAPED_CHAR> | "-" | "+" ) > | <#_WHITESPACE: ( " " | "\t" | "\n" | "\r" | "\u3000") > | <#_QUOTED_CHAR: ( ~[ "\"", "\\" ] | <_ESCAPED_CHAR> ) > +| <#_EXT_ESCAPED_CHAR:( "\\" ["/", "\\"] ) > +| <#_EXT_TERM_CHAR:( <_EXT_ESCAPED_CHAR> | ~["/", "\\"]) > } SKIP : { @@ -1156,7 +1188,8 @@ | | | : Boost -| )* "\""> +| )* "\""> +| : Extension | (<_TERM_CHAR>)* > | )+ ( "." (<_NUM_CHAR>)+ )? )? > | (<_TERM_CHAR>)* "*" ) > @@ -1168,6 +1201,11 @@ TOKEN : { )+ ( "." (<_NUM_CHAR>)+ )? > : DEFAULT } + TOKEN: +{ + | <_WHITESPACE>)+ > +| : DEFAULT +} TOKEN : { @@ -1262,6 +1300,7 @@ ( q=Term(field) + | q=ExtQuery(field) ( boost=)? | q=Query(field) ( boost=)? ) @@ -1276,6 +1315,20 @@ return q; } } +Query ExtQuery(String field): +{ + Query q; + Token extTerm; +} +{ + ( + extTerm= + ) +{ + String termImage=extension.discardEscapeChars(extTerm.image); + return extension.parse(termImage, field); +} +} Query Term(String field) : { Index: contrib/regex/src/java/org/apache/lucene/search/regex/RegexParseExtension.java =================================================================== --- contrib/regex/src/java/org/apache/lucene/search/regex/RegexParseExtension.java (revision 0) +++ contrib/regex/src/java/org/apache/lucene/search/regex/RegexParseExtension.java (revision 0) @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search.regex; + +import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.queryParser.QueryParserExtension; +import org.apache.lucene.search.Query; + +/** + * Standard QueryParser extension for regular expressions. + */ +public final class RegexParseExtension extends QueryParserExtension { + + private final RegexCapabilities capabilities; + + public RegexParseExtension(RegexCapabilities capabilities) { + this.capabilities = capabilities; + } + + /** + * Creates a new {@link RegexQuery} from an embedded query + */ + @Override + public Query parse(String embeddedQuery, String field) throws ParseException { + RegexQuery regexQuery = new RegexQuery(new Term(field, embeddedQuery)); + regexQuery.setRegexImplementation(this.capabilities); + return regexQuery; + } + +} Index: src/test/org/apache/lucene/queryParser/TestQueryParserExtension.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestQueryParserExtension.java (revision 0) +++ src/test/org/apache/lucene/queryParser/TestQueryParserExtension.java (revision 0) @@ -0,0 +1,38 @@ +package org.apache.lucene.queryParser; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import junit.framework.TestCase; + +/** + * Test QueryParserExtension + */ +public class TestQueryParserExtension extends TestCase { + + public void testEscape() { + assertEquals("ab.*[foo](bar) \\\\ \\/ \\\\\\/", QueryParserExtension + .escape("ab.*[foo](bar) \\ / \\/")); + assertEquals("", QueryParserExtension.escape("")); + } + + public void testDiscardEscapeChars() { + assertEquals("ab.*[foo](bar) \\ / \\/", QueryParserExtension + .discardEscapeChars("ab.*[foo](bar) \\\\ \\/ \\\\\\/")); + assertEquals("", QueryParserExtension.discardEscapeChars("")); + } + +} Index: src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestQueryParser.java (revision 833478) +++ src/test/org/apache/lucene/queryParser/TestQueryParser.java (working copy) @@ -48,6 +48,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.FuzzyQuery; @@ -162,10 +163,29 @@ qp.setDefaultOperator(QueryParser.OR_OPERATOR); return qp; } - + public Query getQuery(String query) throws Exception { + return getQuery(query, null, Query.class); + } + public Query getQuery(String query, Analyzer a) throws Exception { - return getParser(a).parse(query); + return getQuery(query, a, Query.class); + } + + public T getQuery(String query, Class assertClass) throws Exception { + return getQuery(query, null, assertClass); + } + + public T getQuery(String aString, Analyzer a, Class assertClass) throws Exception{ + return getQueryAs(getParser(a).parse(aString), assertClass); } + + + public T getQueryAs(Query query, Class assertClass) { + assertTrue(query.getClass() + "is not an instance of " + assertClass.getName(), + assertClass.isAssignableFrom(query.getClass())); + return assertClass.cast(query); + } + public void assertQueryEquals(String query, Analyzer a, String result) throws Exception { @@ -279,9 +299,9 @@ assertQueryEquals("term AND \"phrase phrase\"", null, "+term +\"phrase phrase\""); assertQueryEquals("\"hello there\"", null, "\"hello there\""); - assertTrue(getQuery("a AND b", null) instanceof BooleanQuery); - assertTrue(getQuery("hello", null) instanceof TermQuery); - assertTrue(getQuery("\"hello there\"", null) instanceof PhraseQuery); + assertTrue(getQuery("a AND b") instanceof BooleanQuery); + assertTrue(getQuery("hello") instanceof TermQuery); + assertTrue(getQuery("\"hello there\"") instanceof PhraseQuery); assertQueryEquals("germ term^2.0", null, "germ term^2.0"); assertQueryEquals("(term)^2.0", null, "term^2.0"); @@ -346,20 +366,20 @@ assertQueryEquals("term*germ", null, "term*germ"); assertQueryEquals("term*germ^3", null, "term*germ^3.0"); - assertTrue(getQuery("term*", null) instanceof PrefixQuery); - assertTrue(getQuery("term*^2", null) instanceof PrefixQuery); - assertTrue(getQuery("term~", null) instanceof FuzzyQuery); - assertTrue(getQuery("term~0.7", null) instanceof FuzzyQuery); - FuzzyQuery fq = (FuzzyQuery)getQuery("term~0.7", null); + assertTrue(getQuery("term*") instanceof PrefixQuery); + assertTrue(getQuery("term*^2") instanceof PrefixQuery); + assertTrue(getQuery("term~") instanceof FuzzyQuery); + assertTrue(getQuery("term~0.7") instanceof FuzzyQuery); + FuzzyQuery fq = getQuery("term~0.7", FuzzyQuery.class); assertEquals(0.7f, fq.getMinSimilarity(), 0.1f); assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength()); - fq = (FuzzyQuery)getQuery("term~", null); + fq = getQuery("term~", FuzzyQuery.class); assertEquals(0.5f, fq.getMinSimilarity(), 0.1f); assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength()); assertParseException("term~1.1"); // value > 1, throws exception - assertTrue(getQuery("term*germ", null) instanceof WildcardQuery); + assertTrue(getQuery("term*germ") instanceof WildcardQuery); /* Tests to see that wild card terms are (or are not) properly * lower-cased with propery parser configuration @@ -453,7 +473,7 @@ public void testRange() throws Exception { assertQueryEquals("[ a TO z]", null, "[a TO z]"); - assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod()); + assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, getQuery("[ a TO z]", TermRangeQuery.class).getRewriteMethod()); QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, "field", new SimpleAnalyzer()); qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); @@ -823,7 +843,7 @@ public void assertParseException(String queryString) throws Exception { try { - getQuery(queryString, null); + getQuery(queryString); } catch (ParseException expected) { return; } @@ -1065,5 +1085,49 @@ // expected } } + + public void testQueryParserExtension() throws ParseException, Exception { + QueryParser p = getParser(null); + String defaultField = "field"; + TermQuery query = getQuery("/a\\/abc/", TermQuery.class); + assertEquals(defaultField, query.getTerm().field()); + assertEquals("a/abc", query.getTerm().text()); + query = getQuery("test:/a\\/abc/", TermQuery.class); + assertEquals("test", query.getTerm().field()); + assertEquals("a/abc", query.getTerm().text()); + + query = getQuery("/a\\\\ab\\/c\\\\/", TermQuery.class); + assertEquals(defaultField, query.getTerm().field()); + assertEquals("a\\ab/c\\", query.getTerm().text()); + + query = getQuery("test:/.*lu[c]{1,1}\\\\a[^](java)\\/ /^1.2", + TermQuery.class); + assertEquals("test", query.getTerm().field()); + assertEquals(".*lu[c]{1,1}\\a[^](java)/ ", query.getTerm().text()); + assertEquals(1.2f, query.getBoost()); + + BooleanQuery boolQuery = getQuery( + "test:lucene^1.4 AND [100 TO 1000] OR /.*lu[c]{1,1}\\\\a[^](java)\\/ /^1.2", + BooleanQuery.class); + BooleanClause[] clauses = boolQuery.getClauses(); + assertEquals(3, clauses.length); + assertEquals(TermQuery.class, clauses[0].getQuery().getClass()); + TermQuery clause1 = getQueryAs(clauses[0].getQuery(), TermQuery.class); + assertEquals("lucene", clause1.getTerm().text()); + assertEquals("test", clause1.getTerm().field()); + assertEquals(1.4f, clause1.getBoost()); + + TermRangeQuery clause2 = getQueryAs(clauses[1].getQuery(), + TermRangeQuery.class); + assertEquals("100", clause2.getLowerTerm()); + assertEquals("1000", clause2.getUpperTerm()); + assertEquals(defaultField, clause2.getField()); + + TermQuery clause3 = getQueryAs(clauses[2].getQuery(), TermQuery.class); + assertEquals(".*lu[c]{1,1}\\a[^](java)/ ", clause3.getTerm().text()); + assertEquals(defaultField, clause3.getTerm().field()); + assertEquals(1.2f, clause3.getBoost()); + + } }