Index: lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java (revision 1560368) +++ lucene/core/src/java/org/apache/lucene/search/AutomatonQuery.java (working copy) @@ -128,4 +128,9 @@ buffer.append(ToStringUtils.boost(getBoost())); return buffer.toString(); } + + /** Returns the automaton used to create this query */ + public Automaton getAutomaton() { + return automaton; + } } Index: lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/FakeDocsEnum.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/FakeDocsEnum.java (revision 0) +++ lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/FakeDocsEnum.java (working copy) @@ -0,0 +1,112 @@ +package org.apache.lucene.search.postingshighlight; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; + +// nocommit: document the evil that this thing is... +class FakeDocsEnum extends DocsAndPositionsEnum { + private TokenStream stream; + + private final CharTermAttribute charTermAtt; + private final OffsetAttribute offsetAtt; + private final CharacterRunAutomaton[] matchers; + + private int currentDoc = -1; + private int currentStartOffset = -1; + private int currentEndOffset = -1; + + FakeDocsEnum(TokenStream stream, CharacterRunAutomaton matchers[]) throws IOException { + this.stream = stream; + this.matchers = matchers; + + charTermAtt = stream.addAttribute(CharTermAttribute.class); + offsetAtt = stream.addAttribute(OffsetAttribute.class); + stream.reset(); + } + + @Override + public int nextPosition() throws IOException { + if (stream != null) { + while (stream.incrementToken()) { + for (int i = 0; i < matchers.length; i++) { + if (matchers[i].run(charTermAtt.buffer(), 0, charTermAtt.length())) { + currentStartOffset = offsetAtt.startOffset(); + currentEndOffset = offsetAtt.endOffset(); + return 0; + } + } + } + stream.end(); + stream.close(); + stream = null; + } + // exhausted + currentStartOffset = currentEndOffset = Integer.MAX_VALUE; + return Integer.MAX_VALUE; + } + + @Override + public int freq() throws IOException { + return Integer.MAX_VALUE; // lie + } + + @Override + public int startOffset() throws IOException { + assert currentStartOffset >= 0; + return currentStartOffset; + } + + @Override + public int endOffset() throws IOException { + assert currentEndOffset >= 0; + return currentEndOffset; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override + public int docID() { + return currentDoc; + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + return currentDoc = target; + } + + @Override + public long cost() { + return 0; + } +} Property changes on: lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/FakeDocsEnum.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (revision 1560368) +++ lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (working copy) @@ -30,6 +30,7 @@ import java.util.SortedSet; import java.util.TreeSet; +import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.index.DocsAndPositionsEnum; @@ -43,13 +44,20 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.AutomatonQuery; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InPlaceMergeSorter; import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.automaton.BasicAutomata; +import org.apache.lucene.util.automaton.BasicOperations; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; /** * Simple highlighter that does not analyze fields nor use @@ -335,9 +343,9 @@ throw new IllegalArgumentException("invalid number of maxPassagesIn"); } final IndexReader reader = searcher.getIndexReader(); - query = rewrite(query); + Query rewritten = rewrite(query); SortedSet queryTerms = new TreeSet(); - query.extractTerms(queryTerms); + rewritten.extractTerms(queryTerms); IndexReaderContext readerContext = reader.getContext(); List leaves = readerContext.leaves(); @@ -389,7 +397,15 @@ for(Term term : fieldTerms) { terms[termUpto++] = term.bytes(); } - Map fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages); + + // check if we should do any multitermprocessing + Analyzer analyzer = getIndexAnalyzer(field); + CharacterRunAutomaton automata[] = new CharacterRunAutomaton[0]; + if (analyzer != null) { + automata = extractAutomata(query, field); + } + + Map fieldHighlights = highlightField(field, contents[i], getBreakIterator(field), terms, docids, leaves, numPassages, automata, analyzer); Object[] result = new Object[docids.length]; for (int j = 0; j < docidsIn.length; j++) { @@ -432,8 +448,42 @@ protected char getMultiValuedSeparator(String field) { return ' '; } + + /** + * Returns the analyzer originally used to index the content for {@code field}. + *

+ * This is used to highlight some MultiTermQueries. + * @return Analyzer or null (the default, meaning no special multi-term processing) + */ + protected Analyzer getIndexAnalyzer(String field) { + return null; + } + + /** + * Kinda like extractTerms, but returns automata instead. + */ + private static CharacterRunAutomaton[] extractAutomata(Query query, String field) { + List list = new ArrayList<>(); + if (query instanceof BooleanQuery) { + BooleanClause clauses[] = ((BooleanQuery) query).getClauses(); + for (BooleanClause clause : clauses) { + if (!clause.isProhibited()) { + list.addAll(Arrays.asList(extractAutomata(clause.getQuery(), field))); + } + } + } else if (query instanceof AutomatonQuery) { + AutomatonQuery aq = (AutomatonQuery) query; + if (aq.getField().equals(field)) { + list.add(new CharacterRunAutomaton(aq.getAutomaton())); + } + } else if (query instanceof PrefixQuery) { + String prefix = ((PrefixQuery) query).getPrefix().text(); + list.add(new CharacterRunAutomaton(BasicOperations.concatenate(BasicAutomata.makeString(prefix), BasicAutomata.makeAnyString()))); + } + return list.toArray(new CharacterRunAutomaton[list.size()]); + } - private Map highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List leaves, int maxPassages) throws IOException { + private Map highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List leaves, int maxPassages, CharacterRunAutomaton automata[], Analyzer analyzer) throws IOException { Map highlights = new HashMap(); // reuse in the real sense... for docs in same segment we just advance our old enum @@ -445,6 +495,14 @@ if (fieldFormatter == null) { throw new NullPointerException("PassageFormatter cannot be null"); } + + final BytesRef allTerms[]; + if (automata.length > 0) { + allTerms = new BytesRef[terms.length + 1]; + System.arraycopy(terms, 0, allTerms, 0, terms.length); + } else { + allTerms = terms; + } for (int i = 0; i < docids.length; i++) { String content = contents[i]; @@ -462,9 +520,14 @@ } if (leaf != lastLeaf) { termsEnum = t.iterator(null); - postings = new DocsAndPositionsEnum[terms.length]; + postings = new DocsAndPositionsEnum[allTerms.length]; } - Passage passages[] = highlightDoc(field, terms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages); + if (automata.length > 0) { + FakeDocsEnum dp = new FakeDocsEnum(analyzer.tokenStream(field, content), automata); + dp.advance(doc - subContext.docBase); + postings[terms.length] = dp; + } + Passage passages[] = highlightDoc(field, allTerms, content.length(), bi, doc - subContext.docBase, termsEnum, postings, maxPassages); if (passages.length == 0) { passages = getEmptyHighlight(field, bi, maxPassages); } Index: lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java =================================================================== --- lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (revision 1560368) +++ lucene/highlighter/src/test/org/apache/lucene/search/postingshighlight/TestPostingsHighlighter.java (working copy) @@ -47,6 +47,7 @@ import org.apache.lucene.search.Sort; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.apache.lucene.util.LuceneTestCase; @@ -1119,4 +1120,46 @@ ir.close(); dir.close(); } + + // nocommit: just a trivial test, more tests, test the fakeDocsEnum directly, and so on + public void testOneWildcard() throws Exception { + Directory dir = newDirectory(); + // use simpleanalyzer for more natural tokenization (else "test." is a token) + final Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer); + iwc.setMergePolicy(newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + + FieldType offsetsType = new FieldType(TextField.TYPE_STORED); + offsetsType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + Field body = new Field("body", "", offsetsType); + Document doc = new Document(); + doc.add(body); + + body.setStringValue("This is a test."); + iw.addDocument(doc); + body.setStringValue("Test a one sentence document."); + iw.addDocument(doc); + + IndexReader ir = iw.getReader(); + iw.close(); + + IndexSearcher searcher = newSearcher(ir); + PostingsHighlighter highlighter = new PostingsHighlighter() { + @Override + protected Analyzer getIndexAnalyzer(String field) { + return analyzer; + } + }; + Query query = new WildcardQuery(new Term("body", "te*")); + TopDocs topDocs = searcher.search(query, null, 10, Sort.INDEXORDER); + assertEquals(2, topDocs.totalHits); + String snippets[] = highlighter.highlight("body", query, searcher, topDocs); + assertEquals(2, snippets.length); + assertEquals("This is a test.", snippets[0]); + assertEquals("Test a one sentence document.", snippets[1]); + + ir.close(); + dir.close(); + } }