Index: lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java =================================================================== --- lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (revision 1556710) +++ lucene/queryparser/src/test/org/apache/lucene/queryparser/util/QueryParserTestBase.java (working copy) @@ -558,6 +558,13 @@ assertQueryEquals("((stop))", qpAnalyzer, ""); assertTrue(getQuery("term term term", qpAnalyzer) instanceof BooleanQuery); assertTrue(getQuery("term +stop", qpAnalyzer) instanceof TermQuery); + + CommonQueryParserConfiguration cqpc = getParserConfig(qpAnalyzer); + setDefaultOperatorAND(cqpc); + assertQueryEquals(cqpc, "field", "term phrase term", + "+term +(+phrase1 +phrase2) +term"); + assertQueryEquals(cqpc, "field", "phrase", + "+phrase1 +phrase2"); } public void testRange() throws Exception { Index: lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java =================================================================== --- lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java (revision 1556710) +++ lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java (working copy) @@ -18,7 +18,9 @@ */ import java.io.IOException; + import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedList; import java.util.List; @@ -29,9 +31,12 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.config.QueryConfigHandler; +import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode.Modifier; import org.apache.lucene.queryparser.flexible.core.nodes.NoTokenFoundQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QuotedFieldQueryNode; @@ -40,6 +45,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.TokenizedPhraseQueryNode; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.ConfigurationKeys; +import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler.Operator; import org.apache.lucene.queryparser.flexible.standard.nodes.MultiPhraseQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.RegexpQueryNode; import org.apache.lucene.queryparser.flexible.standard.nodes.StandardBooleanQueryNode; @@ -72,6 +78,8 @@ private Analyzer analyzer; private boolean positionIncrementsEnabled; + + private Operator defaultOperator; public AnalyzerQueryNodeProcessor() { // empty constructor @@ -85,7 +93,9 @@ this.analyzer = analyzer; this.positionIncrementsEnabled = false; Boolean positionIncrementsEnabled = getQueryConfigHandler().get(ConfigurationKeys.ENABLE_POSITION_INCREMENTS); - + Operator defaultOperator = getQueryConfigHandler().get(ConfigurationKeys.DEFAULT_OPERATOR); + this.defaultOperator = defaultOperator != null ? defaultOperator : Operator.OR; + if (positionIncrementsEnabled != null) { this.positionIncrementsEnabled = positionIncrementsEnabled; } @@ -93,7 +103,6 @@ if (this.analyzer != null) { return super.process(queryTree); } - } return queryTree; @@ -119,197 +128,250 @@ int positionCount = 0; boolean severalTokensAtSamePosition = false; - try (TokenStream source = this.analyzer.tokenStream(field, text)) { - source.reset(); - buffer = new CachingTokenFilter(source); - - if (buffer.hasAttribute(PositionIncrementAttribute.class)) { - posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); - } - - try { - - while (buffer.incrementToken()) { - numTokens++; - int positionIncrement = (posIncrAtt != null) ? posIncrAtt - .getPositionIncrement() : 1; - if (positionIncrement != 0) { - positionCount += positionIncrement; - - } else { - severalTokensAtSamePosition = true; + try { + try (TokenStream source = this.analyzer.tokenStream(field, text)) { + source.reset(); + buffer = new CachingTokenFilter(source); + + if (buffer.hasAttribute(PositionIncrementAttribute.class)) { + posIncrAtt = buffer.getAttribute(PositionIncrementAttribute.class); + } + + try { + + while (buffer.incrementToken()) { + numTokens++; + int positionIncrement = (posIncrAtt != null) ? posIncrAtt + .getPositionIncrement() : 1; + if (positionIncrement != 0) { + positionCount += positionIncrement; + + } else { + severalTokensAtSamePosition = true; + } + } - + + } catch (IOException e) { + // ignore } - } catch (IOException e) { - // ignore + throw new RuntimeException(e); } - } catch (IOException e) { - throw new RuntimeException(e); - } - - // rewind the buffer stream - buffer.reset(); - - if (!buffer.hasAttribute(CharTermAttribute.class)) { - return new NoTokenFoundQueryNode(); - } - - CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class); - - if (numTokens == 0) { - return new NoTokenFoundQueryNode(); - - } else if (numTokens == 1) { - String term = null; - try { - boolean hasNext; - hasNext = buffer.incrementToken(); - assert hasNext == true; - term = termAtt.toString(); - - } catch (IOException e) { - // safe to ignore, because we know the number of tokens + + // rewind the buffer stream + buffer.reset(); + + if (!buffer.hasAttribute(CharTermAttribute.class)) { + return new NoTokenFoundQueryNode(); } - - fieldNode.setText(term); - - return fieldNode; - - } else if (severalTokensAtSamePosition || !(node instanceof QuotedFieldQueryNode)) { - if (positionCount == 1 || !(node instanceof QuotedFieldQueryNode)) { - // no phrase query: - LinkedList children = new LinkedList(); - - for (int i = 0; i < numTokens; i++) { - String term = null; - try { - boolean hasNext = buffer.incrementToken(); - assert hasNext == true; - term = termAtt.toString(); - - } catch (IOException e) { - // safe to ignore, because we know the number of tokens + + CharTermAttribute termAtt = buffer.getAttribute(CharTermAttribute.class); + + if (numTokens == 0) { + return new NoTokenFoundQueryNode(); + + } else if (numTokens == 1) { + String term = null; + try { + boolean hasNext; + hasNext = buffer.incrementToken(); + assert hasNext == true; + term = termAtt.toString(); + + } catch (IOException e) { + // safe to ignore, because we know the number of tokens + } + + fieldNode.setText(term); + + return fieldNode; + + } else if (severalTokensAtSamePosition || !(node instanceof QuotedFieldQueryNode)) { + if (positionCount == 1 || !(node instanceof QuotedFieldQueryNode)) { + // no phrase query: + + if (positionCount == 1) { + // simple case: only one position, with synonyms + LinkedList children = new LinkedList(); + + for (int i = 0; i < numTokens; i++) { + String term = null; + try { + boolean hasNext = buffer.incrementToken(); + assert hasNext == true; + term = termAtt.toString(); + + } catch (IOException e) { + // safe to ignore, because we know the number of tokens + } + + children.add(new FieldQueryNode(field, term, -1, -1)); + + } + return new GroupQueryNode( + new StandardBooleanQueryNode(children, positionCount==1)); + } else { + // multiple positions + QueryNode q = new StandardBooleanQueryNode(Collections.emptyList(),false); + QueryNode currentQuery = null; + for (int i = 0; i < numTokens; i++) { + String term = null; + try { + boolean hasNext = buffer.incrementToken(); + assert hasNext == true; + term = termAtt.toString(); + } catch (IOException e) { + // safe to ignore, because we know the number of tokens + } + if (posIncrAtt != null && posIncrAtt.getPositionIncrement() == 0) { + if (!(currentQuery instanceof BooleanQueryNode)) { + QueryNode t = currentQuery; + currentQuery = new StandardBooleanQueryNode(Collections.emptyList(), true); + ((BooleanQueryNode)currentQuery).add(t); + } + ((BooleanQueryNode)currentQuery).add(new FieldQueryNode(field, term, -1, -1)); + } else { + if (currentQuery != null) { + if (this.defaultOperator == Operator.OR) { + q.add(currentQuery); + } else { + q.add(new ModifierQueryNode(currentQuery, Modifier.MOD_REQ)); + } + } + currentQuery = new FieldQueryNode(field, term, -1, -1); + } + } + if (this.defaultOperator == Operator.OR) { + q.add(currentQuery); + } else { + q.add(new ModifierQueryNode(currentQuery, Modifier.MOD_REQ)); + } + + if (q instanceof BooleanQueryNode) { + q = new GroupQueryNode(q); + } + return q; } - - children.add(new FieldQueryNode(field, term, -1, -1)); - + } else { + // phrase query: + MultiPhraseQueryNode mpq = new MultiPhraseQueryNode(); + + List multiTerms = new ArrayList(); + int position = -1; + int i = 0; + int termGroupCount = 0; + for (; i < numTokens; i++) { + String term = null; + int positionIncrement = 1; + try { + boolean hasNext = buffer.incrementToken(); + assert hasNext == true; + term = termAtt.toString(); + if (posIncrAtt != null) { + positionIncrement = posIncrAtt.getPositionIncrement(); + } + + } catch (IOException e) { + // safe to ignore, because we know the number of tokens + } + + if (positionIncrement > 0 && multiTerms.size() > 0) { + + for (FieldQueryNode termNode : multiTerms) { + + if (this.positionIncrementsEnabled) { + termNode.setPositionIncrement(position); + } else { + termNode.setPositionIncrement(termGroupCount); + } + + mpq.add(termNode); + + } + + // Only increment once for each "group" of + // terms that were in the same position: + termGroupCount++; + + multiTerms.clear(); + + } + + position += positionIncrement; + multiTerms.add(new FieldQueryNode(field, term, -1, -1)); + + } + + for (FieldQueryNode termNode : multiTerms) { + + if (this.positionIncrementsEnabled) { + termNode.setPositionIncrement(position); + + } else { + termNode.setPositionIncrement(termGroupCount); + } + + mpq.add(termNode); + + } + + return mpq; + } - return new GroupQueryNode( - new StandardBooleanQueryNode(children, positionCount==1)); + } else { - // phrase query: - MultiPhraseQueryNode mpq = new MultiPhraseQueryNode(); - - List multiTerms = new ArrayList(); + + TokenizedPhraseQueryNode pq = new TokenizedPhraseQueryNode(); + int position = -1; - int i = 0; - int termGroupCount = 0; - for (; i < numTokens; i++) { + + for (int i = 0; i < numTokens; i++) { String term = null; int positionIncrement = 1; + try { boolean hasNext = buffer.incrementToken(); assert hasNext == true; term = termAtt.toString(); + if (posIncrAtt != null) { positionIncrement = posIncrAtt.getPositionIncrement(); } - + } catch (IOException e) { // safe to ignore, because we know the number of tokens } - - if (positionIncrement > 0 && multiTerms.size() > 0) { - - for (FieldQueryNode termNode : multiTerms) { - - if (this.positionIncrementsEnabled) { - termNode.setPositionIncrement(position); - } else { - termNode.setPositionIncrement(termGroupCount); - } - - mpq.add(termNode); - - } - - // Only increment once for each "group" of - // terms that were in the same position: - termGroupCount++; - - multiTerms.clear(); - - } - - position += positionIncrement; - multiTerms.add(new FieldQueryNode(field, term, -1, -1)); - - } - - for (FieldQueryNode termNode : multiTerms) { - + + FieldQueryNode newFieldNode = new FieldQueryNode(field, term, -1, -1); + if (this.positionIncrementsEnabled) { - termNode.setPositionIncrement(position); - + position += positionIncrement; + newFieldNode.setPositionIncrement(position); + } else { - termNode.setPositionIncrement(termGroupCount); + newFieldNode.setPositionIncrement(i); } - - mpq.add(termNode); - + + pq.add(newFieldNode); + } - - return mpq; - + + return pq; + } - - } else { - - TokenizedPhraseQueryNode pq = new TokenizedPhraseQueryNode(); - - int position = -1; - - for (int i = 0; i < numTokens; i++) { - String term = null; - int positionIncrement = 1; - + } finally { + if (buffer != null) { try { - boolean hasNext = buffer.incrementToken(); - assert hasNext == true; - term = termAtt.toString(); - - if (posIncrAtt != null) { - positionIncrement = posIncrAtt.getPositionIncrement(); - } - + buffer.close(); } catch (IOException e) { - // safe to ignore, because we know the number of tokens + // safe to ignore } - - FieldQueryNode newFieldNode = new FieldQueryNode(field, term, -1, -1); - - if (this.positionIncrementsEnabled) { - position += positionIncrement; - newFieldNode.setPositionIncrement(position); - - } else { - newFieldNode.setPositionIncrement(i); - } - - pq.add(newFieldNode); - } - - return pq; - } - } return node; - } @Override Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1556710) +++ lucene/CHANGES.txt (working copy) @@ -136,6 +136,9 @@ * LUCENE-5361: Fixed handling of query boosts in FastVectorHighlighter. (Nik Everett via Adrien Grand) + +* LUCENE-5344: Flexible StandardQueryParser behaves differently than + ClassicQueryParser. (Adriano Crestani) API Changes