Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/AbstractTestCase.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/AbstractTestCase.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/AbstractTestCase.java (revision 0) @@ -0,0 +1,345 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.util.Collection; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +public abstract class AbstractTestCase extends TestCase { + + protected final String F = "f"; + protected final String F1 = "f1"; + protected final String F2 = "f2"; + protected Directory dir; + protected Analyzer analyzerW; + protected Analyzer analyzerB; + protected IndexReader reader; + protected QueryParser paW; + protected QueryParser paB; + + protected static final String[] shortMVValues = { + "a b c", + "", // empty data in multi valued field + "d e" + }; + + protected static final String[] longMVValues = { + "Followings are the examples of customizable parameters and actual examples of customization:", + "The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically" + }; + + // test data for LUCENE-1448 bug + protected static final String[] biMVValues = { + "\nLucene/Solr does not require such additional hardware.", + "\nWhen you talk about processing speed, the" + }; + + protected void setUp() throws Exception { + analyzerW = new WhitespaceAnalyzer(); + analyzerB = new BigramAnalyzer(); + paW = new QueryParser( F, analyzerW ); + paB = new QueryParser( F, analyzerB ); + dir = new RAMDirectory(); + } + + protected void tearDown() throws Exception { + if( reader != null ){ + reader.close(); + reader = null; + } + } + + protected Query tq( String text ){ + return tq( 1F, text ); + } + + protected Query tq( float boost, String text ){ + return tq( boost, F, text ); + } + + protected Query tq( String field, String text ){ + return tq( 1F, field, text ); + } + + protected Query tq( float boost, String field, String text ){ + Query query = new TermQuery( new Term( field, text ) ); + query.setBoost( boost ); + return query; + } + + protected Query pqF( String... texts ){ + return pqF( 1F, texts ); + } + + protected Query pqF( float boost, String... texts ){ + return pqF( boost, 0, texts ); + } + + protected Query pqF( float boost, int slop, String... texts ){ + return pq( boost, slop, F, texts ); + } + + protected Query pq( String field, String... texts ){ + return pq( 1F, 0, field, texts ); + } + + protected Query pq( float boost, String field, String... texts ){ + return pq( boost, 0, field, texts ); + } + + protected Query pq( float boost, int slop, String field, String... texts ){ + PhraseQuery query = new PhraseQuery(); + for( String text : texts ){ + query.add( new Term( field, text ) ); + } + query.setBoost( boost ); + query.setSlop( slop ); + return query; + } + + protected void assertCollectionQueries( Collection actual, Query... expected ){ + assertEquals( expected.length, actual.size() ); + for( Query query : expected ){ + assertTrue( actual.contains( query ) ); + } + } + + static class BigramAnalyzer extends Analyzer { + public TokenStream tokenStream(String fieldName, Reader reader) { + return new BasicNGramTokenizer( reader ); + } + } + + static class BasicNGramTokenizer extends Tokenizer { + + public static final int DEFAULT_N_SIZE = 2; + public static final String DEFAULT_DELIMITERS = " \t\n.,"; + private final int n; + private final String delimiters; + private int startTerm; + private int lenTerm; + private int startOffset; + private int nextStartOffset; + private int ch; + private String snippet; + private StringBuilder snippetBuffer; + private static final int BUFFER_SIZE = 4096; + private char[] charBuffer; + private int charBufferIndex; + private int charBufferLen; + + public BasicNGramTokenizer( Reader in ){ + this( in, DEFAULT_N_SIZE ); + } + + public BasicNGramTokenizer( Reader in, int n ){ + this( in, n, DEFAULT_DELIMITERS ); + } + + public BasicNGramTokenizer( Reader in, String delimiters ){ + this( in, DEFAULT_N_SIZE, delimiters ); + } + + public BasicNGramTokenizer( Reader in, int n, String delimiters ){ + input = in; + this.n = n; + this.delimiters = delimiters; + startTerm = 0; + nextStartOffset = 0; + snippet = null; + snippetBuffer = new StringBuilder(); + charBuffer = new char[BUFFER_SIZE]; + charBufferIndex = BUFFER_SIZE; + charBufferLen = 0; + ch = 0; + } + + public Token next( Token reusableToken ) throws IOException { + if( !getNextPartialSnippet() ) + return null; + reusableToken.reinit( snippet, startTerm, lenTerm, startOffset, startOffset + lenTerm ); + return reusableToken; + } + + public int getFinalOffset() { + return nextStartOffset; + } + + protected boolean getNextPartialSnippet() throws IOException { + if( snippet != null && snippet.length() >= startTerm + 1 + n ){ + startTerm++; + startOffset++; + lenTerm = n; + return true; + } + return getNextSnippet(); + } + + protected boolean getNextSnippet() throws IOException { + startTerm = 0; + startOffset = nextStartOffset; + snippetBuffer.delete( 0, snippetBuffer.length() ); + while( true ){ + if( ch != -1 ) + ch = readCharFromBuffer(); + if( ch == -1 ) break; + else if( !isDelimiter( ch ) ) + snippetBuffer.append( (char)ch ); + else if( snippetBuffer.length() > 0 ) + break; + else + startOffset++; + } + if( snippetBuffer.length() == 0 ) + return false; + snippet = snippetBuffer.toString(); + lenTerm = snippet.length() >= n ? n : snippet.length(); + return true; + } + + protected int readCharFromBuffer() throws IOException { + if( charBufferIndex >= charBufferLen ){ + charBufferLen = input.read( charBuffer ); + if( charBufferLen == -1 ){ + return -1; + } + charBufferIndex = 0; + } + int c = (int)charBuffer[charBufferIndex++]; + nextStartOffset++; + return c; + } + + protected boolean isDelimiter( int c ){ + return delimiters.indexOf( c ) >= 0; + } + } + + protected void make1d1fIndex( String value ) throws Exception { + make1dmfIndex( value ); + } + + protected void make1d1fIndexB( String value ) throws Exception { + make1dmfIndexB( value ); + } + + protected void make1dmfIndex( String... values ) throws Exception { + make1dmfIndex( analyzerW, values ); + } + + protected void make1dmfIndexB( String... values ) throws Exception { + make1dmfIndex( analyzerB, values ); + } + + protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception { + IndexWriter writer = new IndexWriter( dir, analyzer, true, MaxFieldLength.LIMITED ); + Document doc = new Document(); + for( String value: values ) + doc.add( new Field( F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) ); + writer.addDocument( doc ); + writer.close(); + + reader = IndexReader.open( dir ); + } + + protected void makeIndexShortMV() throws Exception { + + // 012345 + // "a b c" + // 0 1 2 + + // "" + + // 6789 + // "d e" + // 3 4 + make1dmfIndex( shortMVValues ); + } + + protected void makeIndexLongMV() throws Exception { + // 11111111112222222222333333333344444444445555555555666666666677777777778888888888999 + // 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012 + // Followings are the examples of customizable parameters and actual examples of customization: + // 0 1 2 3 4 5 6 7 8 9 10 11 + + // 1 2 + // 999999900000000001111111111222222222233333333334444444444555555555566666666667777777777888888888899999999990000000000111111111122 + // 345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901 + // The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically + // 12 13 (14) (15) 16 17 18 19 20 21 22 23 (24) (25) 26 27 28 29 30 31 32 33 34 + + make1dmfIndex( longMVValues ); + } + + protected void makeIndexLongMVB() throws Exception { + // "*" ... LF + + // 1111111111222222222233333333334444444444555555 + // 01234567890123456789012345678901234567890123456789012345 + // *Lucene/Solr does not require such additional hardware. + // Lu 0 do 10 re 15 su 21 na 31 + // uc 1 oe 11 eq 16 uc 22 al 32 + // ce 2 es 12 qu 17 ch 23 ha 33 + // en 3 no 13 ui 18 ad 24 ar 34 + // ne 4 ot 14 ir 19 dd 25 rd 35 + // e/ 5 re 20 di 26 dw 36 + // /S 6 it 27 wa 37 + // So 7 ti 28 ar 38 + // ol 8 io 29 re 39 + // lr 9 on 30 + + // 5555666666666677777777778888888888999999999 + // 6789012345678901234567890123456789012345678 + // *When you talk about processing speed, the + // Wh 40 ab 48 es 56 th 65 + // he 41 bo 49 ss 57 he 66 + // en 42 ou 50 si 58 + // yo 43 ut 51 in 59 + // ou 44 pr 52 ng 60 + // ta 45 ro 53 sp 61 + // al 46 oc 54 pe 62 + // lk 47 ce 55 ee 63 + // ed 64 + + make1dmfIndexB( biMVValues ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/AbstractTestCase.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldTermStackTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldTermStackTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldTermStackTest.java (revision 0) @@ -0,0 +1,161 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause.Occur; + +public class FieldTermStackTest extends AbstractTestCase { + + public void test1Term() throws Exception { + makeIndex(); + + FieldQuery fq = new FieldQuery( tq( "a" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 6, stack.termList.size() ); + assertEquals( "a(0,1,0)", stack.pop().toString() ); + assertEquals( "a(2,3,1)", stack.pop().toString() ); + assertEquals( "a(4,5,2)", stack.pop().toString() ); + assertEquals( "a(12,13,6)", stack.pop().toString() ); + assertEquals( "a(28,29,14)", stack.pop().toString() ); + assertEquals( "a(32,33,16)", stack.pop().toString() ); + } + + public void test2Terms() throws Exception { + makeIndex(); + + BooleanQuery query = new BooleanQuery(); + query.add( tq( "b" ), Occur.SHOULD ); + query.add( tq( "c" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 8, stack.termList.size() ); + assertEquals( "b(6,7,3)", stack.pop().toString() ); + assertEquals( "b(8,9,4)", stack.pop().toString() ); + assertEquals( "c(10,11,5)", stack.pop().toString() ); + assertEquals( "b(14,15,7)", stack.pop().toString() ); + assertEquals( "b(16,17,8)", stack.pop().toString() ); + assertEquals( "c(18,19,9)", stack.pop().toString() ); + assertEquals( "b(26,27,13)", stack.pop().toString() ); + assertEquals( "b(30,31,15)", stack.pop().toString() ); + } + + public void test1Phrase() throws Exception { + makeIndex(); + + FieldQuery fq = new FieldQuery( pqF( "c", "d" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 3, stack.termList.size() ); + assertEquals( "c(10,11,5)", stack.pop().toString() ); + assertEquals( "c(18,19,9)", stack.pop().toString() ); + assertEquals( "d(20,21,10)", stack.pop().toString() ); + } + + private void makeIndex() throws Exception { + // 111111111122222 + // 0123456789012345678901234 (offsets) + // a a a b b c a b b c d e f + // 0 1 2 3 4 5 6 7 8 9101112 (position) + String value1 = "a a a b b c a b b c d e f"; + // 222233333 + // 678901234 (offsets) + // b a b a f + //1314151617 (position) + String value2 = "b a b a f"; + + make1dmfIndex( value1, value2 ); + } + + public void test1TermB() throws Exception { + makeIndexB(); + + FieldQuery fq = new FieldQuery( tq( "ab" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 2, stack.termList.size() ); + assertEquals( "ab(2,4,2)", stack.pop().toString() ); + assertEquals( "ab(6,8,6)", stack.pop().toString() ); + } + + public void test2TermsB() throws Exception { + makeIndexB(); + + BooleanQuery query = new BooleanQuery(); + query.add( tq( "bc" ), Occur.SHOULD ); + query.add( tq( "ef" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 3, stack.termList.size() ); + assertEquals( "bc(4,6,4)", stack.pop().toString() ); + assertEquals( "bc(8,10,8)", stack.pop().toString() ); + assertEquals( "ef(11,13,11)", stack.pop().toString() ); + } + + public void test1PhraseB() throws Exception { + makeIndexB(); + + FieldQuery fq = new FieldQuery( pqF( "ab", "bb" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 4, stack.termList.size() ); + assertEquals( "ab(2,4,2)", stack.pop().toString() ); + assertEquals( "bb(3,5,3)", stack.pop().toString() ); + assertEquals( "ab(6,8,6)", stack.pop().toString() ); + assertEquals( "bb(7,9,7)", stack.pop().toString() ); + } + + private void makeIndexB() throws Exception { + // 1 11 11 + // 01 12 23 34 45 56 67 78 89 90 01 12 (offsets) + // aa|aa|ab|bb|bc|ca|ab|bb|bc|cd|de|ef + // 0 1 2 3 4 5 6 7 8 9 10 11 (position) + String value = "aaabbcabbcdef"; + + make1dmfIndexB( value ); + } + + public void test1PhraseShortMV() throws Exception { + makeIndexShortMV(); + + FieldQuery fq = new FieldQuery( tq( "d" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "d(6,7,3)", stack.pop().toString() ); + } + + public void test1PhraseLongMV() throws Exception { + makeIndexLongMV(); + + FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 4, stack.termList.size() ); + assertEquals( "search(102,108,14)", stack.pop().toString() ); + assertEquals( "engines(109,116,15)", stack.pop().toString() ); + assertEquals( "search(157,163,24)", stack.pop().toString() ); + assertEquals( "engines(164,171,25)", stack.pop().toString() ); + } + + public void test1PhraseMVB() throws Exception { + makeIndexLongMVB(); + + FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed" + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 4, stack.termList.size() ); + assertEquals( "sp(88,90,61)", stack.pop().toString() ); + assertEquals( "pe(89,91,62)", stack.pop().toString() ); + assertEquals( "ee(90,92,63)", stack.pop().toString() ); + assertEquals( "ed(91,93,64)", stack.pop().toString() ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldTermStackTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldPhraseListTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldPhraseListTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldPhraseListTest.java (revision 0) @@ -0,0 +1,177 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause.Occur; + +public class FieldPhraseListTest extends AbstractTestCase { + + public void test1TermIndex() throws Exception { + make1d1fIndex( "a" ); + + FieldQuery fq = new FieldQuery( tq( "a" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() ); + + fq = new FieldQuery( tq( "b" ), true, true ); + stack = new FieldTermStack( reader, 0, F, fq ); + fpl = new FieldPhraseList( stack, fq ); + assertEquals( 0, fpl.phraseList.size() ); + } + + public void test2TermsIndex() throws Exception { + make1d1fIndex( "a a" ); + + FieldQuery fq = new FieldQuery( tq( "a" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 2, fpl.phraseList.size() ); + assertEquals( "a(1.0)((0,1))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 1 ).toString() ); + } + + public void test1PhraseIndex() throws Exception { + make1d1fIndex( "a b" ); + + FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "ab(1.0)((0,3))", fpl.phraseList.get( 0 ).toString() ); + + fq = new FieldQuery( tq( "b" ), true, true ); + stack = new FieldTermStack( reader, 0, F, fq ); + fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "b(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() ); + } + + public void test1PhraseIndexB() throws Exception { + // 01 12 23 34 45 56 67 78 (offsets) + // bb|bb|ba|ac|cb|ba|ab|bc + // 0 1 2 3 4 5 6 7 (positions) + make1d1fIndexB( "bbbacbabc" ); + + FieldQuery fq = new FieldQuery( pqF( "ba", "ac" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "baac(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() ); + } + + public void test2Terms1PhraseIndex() throws Exception { + make1d1fIndex( "c a a b" ); + + // phraseHighlight = true + FieldQuery fq = new FieldQuery( pqF( "a", "b" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 0 ).toString() ); + + // phraseHighlight = false + fq = new FieldQuery( pqF( "a", "b" ), false, true ); + stack = new FieldTermStack( reader, 0, F, fq ); + fpl = new FieldPhraseList( stack, fq ); + assertEquals( 2, fpl.phraseList.size() ); + assertEquals( "a(1.0)((2,3))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( "ab(1.0)((4,7))", fpl.phraseList.get( 1 ).toString() ); + } + + public void testPhraseSlop() throws Exception { + make1d1fIndex( "c a a b c" ); + + FieldQuery fq = new FieldQuery( pqF( 2F, 1, "a", "c" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "ac(2.0)((4,5)(8,9))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 4, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 9, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void test2PhrasesOverlap() throws Exception { + make1d1fIndex( "d a b c d" ); + + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "b" ), Occur.SHOULD ); + query.add( pqF( "b", "c" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "abc(1.0)((2,7))", fpl.phraseList.get( 0 ).toString() ); + } + + public void test3TermsPhrase() throws Exception { + make1d1fIndex( "d a b a b c d" ); + + FieldQuery fq = new FieldQuery( pqF( "a", "b", "c" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "abc(1.0)((6,11))", fpl.phraseList.get( 0 ).toString() ); + } + + public void testSearchLongestPhrase() throws Exception { + make1d1fIndex( "d a b d c a b c" ); + + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "b" ), Occur.SHOULD ); + query.add( pqF( "a", "b", "c" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 2, fpl.phraseList.size() ); + assertEquals( "ab(1.0)((2,5))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( "abc(1.0)((10,15))", fpl.phraseList.get( 1 ).toString() ); + } + + public void test1PhraseShortMV() throws Exception { + makeIndexShortMV(); + + FieldQuery fq = new FieldQuery( tq( "d" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "d(1.0)((6,7))", fpl.phraseList.get( 0 ).toString() ); + } + + public void test1PhraseLongMV() throws Exception { + makeIndexLongMV(); + + FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 2, fpl.phraseList.size() ); + assertEquals( "searchengines(1.0)((102,116))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( "searchengines(1.0)((157,171))", fpl.phraseList.get( 1 ).toString() ); + } + + public void test1PhraseLongMVB() throws Exception { + makeIndexLongMVB(); + + FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed" + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "sppeeeed(1.0)((88,93))", fpl.phraseList.get( 0 ).toString() ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldPhraseListTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/IndexTimeSynonymTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/IndexTimeSynonymTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/IndexTimeSynonymTest.java (revision 0) @@ -0,0 +1,308 @@ +package org.apache.lucene.search.highlight2; + +import java.io.IOException; +import java.io.Reader; +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.BooleanClause.Occur; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class IndexTimeSynonymTest extends AbstractTestCase { + + public void testFieldTermStackIndex1wSearch1term() throws Exception { + makeIndex1w(); + + FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "Mac(11,20,3)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex1wSearch2terms() throws Exception { + makeIndex1w(); + + BooleanQuery bq = new BooleanQuery(); + bq.add( tq( "Mac" ), Occur.SHOULD ); + bq.add( tq( "MacBook" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( bq, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 2, stack.termList.size() ); + Set expectedSet = new HashSet(); + expectedSet.add( "Mac(11,20,3)" ); + expectedSet.add( "MacBook(11,20,3)" ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + } + + public void testFieldTermStackIndex1w2wSearch1term() throws Exception { + makeIndex1w2w(); + + FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "pc(3,5,1)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception { + makeIndex1w2w(); + + FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 2, stack.termList.size() ); + assertEquals( "personal(3,5,1)", stack.pop().toString() ); + assertEquals( "computer(3,5,2)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex1w2wSearch1partial() throws Exception { + makeIndex1w2w(); + + FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "computer(3,5,2)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception { + makeIndex1w2w(); + + BooleanQuery bq = new BooleanQuery(); + bq.add( tq( "pc" ), Occur.SHOULD ); + bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( bq, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 3, stack.termList.size() ); + Set expectedSet = new HashSet(); + expectedSet.add( "pc(3,5,1)" ); + expectedSet.add( "personal(3,5,1)" ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + assertEquals( "computer(3,5,2)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex2w1wSearch1term() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "pc(3,20,1)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 2, stack.termList.size() ); + assertEquals( "personal(3,20,1)", stack.pop().toString() ); + assertEquals( "computer(3,20,2)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex2w1wSearch1partial() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 1, stack.termList.size() ); + assertEquals( "computer(3,20,2)", stack.pop().toString() ); + } + + public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception { + makeIndex2w1w(); + + BooleanQuery bq = new BooleanQuery(); + bq.add( tq( "pc" ), Occur.SHOULD ); + bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( bq, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + assertEquals( 3, stack.termList.size() ); + Set expectedSet = new HashSet(); + expectedSet.add( "pc(3,20,1)" ); + expectedSet.add( "personal(3,20,1)" ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + assertTrue( expectedSet.contains( stack.pop().toString() ) ); + assertEquals( "computer(3,20,2)", stack.pop().toString() ); + } + + public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception { + makeIndex1w2w(); + + FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception { + makeIndex1w2w(); + + FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception { + makeIndex1w2w(); + + BooleanQuery bq = new BooleanQuery(); + bq.add( tq( "pc" ), Occur.SHOULD ); + bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( bq, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex2w1wSearch1term() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception { + makeIndex2w1w(); + + FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception { + makeIndex2w1w(); + + BooleanQuery bq = new BooleanQuery(); + bq.add( tq( "pc" ), Occur.SHOULD ); + bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); + FieldQuery fq = new FieldQuery( bq, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + assertEquals( 1, fpl.phraseList.size() ); + assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 ); + assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); + assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); + } + + private void makeIndex1w() throws Exception { + // 11111111112 + // 012345678901234567890 + // I'll buy a Macintosh + // Mac + // MacBook + // 0 1 2 3 + makeSynonymIndex( "I'll buy a Macintosh", + t("I'll",0,4), + t("buy",5,8), + t("a",9,10), + t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0)); + } + + private void makeIndex1w2w() throws Exception { + // 1111111 + // 01234567890123456 + // My pc was broken + // personal computer + // 0 1 2 3 + makeSynonymIndex( "My pc was broken", + t("My",0,2), + t("pc",3,5),t("personal",3,5,0),t("computer",3,5), + t("was",6,9), + t("broken",10,16)); + } + + private void makeIndex2w1w() throws Exception { + // 1111111111222222222233 + // 01234567890123456789012345678901 + // My personal computer was broken + // pc + // 0 1 2 3 4 + makeSynonymIndex( "My personal computer was broken", + t("My",0,2), + t("personal",3,20),t("pc",3,20,0),t("computer",3,20), + t("was",21,24), + t("broken",25,31)); + } + + void makeSynonymIndex( String value, Token... tokens ) throws Exception { + Analyzer analyzer = new TokenArrayAnalyzer( tokens ); + make1dmfIndex( analyzer, value ); + } + + public static Token t( String text, int startOffset, int endOffset ){ + return t( text, startOffset, endOffset, 1 ); + } + + public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){ + Token token = new Token( text, startOffset, endOffset ); + token.setPositionIncrement( positionIncrement ); + return token; + } + + public static class TokenArrayAnalyzer extends Analyzer { + Token[] tokens; + public TokenArrayAnalyzer( Token... tokens ){ + this.tokens = tokens; + } + public TokenStream tokenStream(String fieldName, Reader reader) { + return new TokenStream(){ + int p = 0; + public Token next( Token reusableToken ) throws IOException { + if( p >= tokens.length ) return null; + return tokens[p++]; + } + }; + } + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/IndexTimeSynonymTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilderTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilderTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilderTest.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Query; + +public class ScoreOrderFragmentsBuilderTest extends AbstractTestCase { + + public void test3Frags() throws Exception { + FieldFragList ffl = ffl( "a c", "a b b b b b b b b b b b a b a b b b b b c a a b b" ); + ScoreOrderFragmentsBuilder sofb = new ScoreOrderFragmentsBuilder(); + String[] f = sofb.createFragments( reader, 0, F, ffl, 3 ); + assertEquals( 3, f.length ); + // check score order + assertEquals( "c a a b b", f[0] ); + assertEquals( "b b a b a b b b b b ", f[1] ); + assertEquals( "a b b b b b b b b b ", f[2] ); + } + + private FieldFragList ffl( String queryValue, String indexValue ) throws Exception { + make1d1fIndex( indexValue ); + Query query = paW.parse( queryValue ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + return new SimpleFragListBuilder().createFieldFragList( fpl, 20 ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilderTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragmentsBuilderTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragmentsBuilderTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragmentsBuilderTest.java (revision 0) @@ -0,0 +1,99 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Query; + +public class SimpleFragmentsBuilderTest extends AbstractTestCase { + + public void test1TermIndex() throws Exception { + FieldFragList ffl = ffl( "a", "a" ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + assertEquals( "a", sfb.createFragment( reader, 0, F, ffl ) ); + + // change tags + sfb = new SimpleFragmentsBuilder( new String[]{ "[" }, new String[]{ "]" } ); + assertEquals( "[a]", sfb.createFragment( reader, 0, F, ffl ) ); + } + + public void test2Frags() throws Exception { + FieldFragList ffl = ffl( "a", "a b b b b b b b b b b b a b a b" ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + String[] f = sfb.createFragments( reader, 0, F, ffl, 3 ); + // 3 snippets requested, but should be 2 + assertEquals( 2, f.length ); + assertEquals( "a b b b b b b b b b ", f[0] ); + assertEquals( "b b a b a b", f[1] ); + } + + public void test3Frags() throws Exception { + FieldFragList ffl = ffl( "a c", "a b b b b b b b b b b b a b a b b b b b c a a b b" ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + String[] f = sfb.createFragments( reader, 0, F, ffl, 3 ); + assertEquals( 3, f.length ); + assertEquals( "a b b b b b b b b b ", f[0] ); + assertEquals( "b b a b a b b b b b ", f[1] ); + assertEquals( "c a a b b", f[2] ); + } + + private FieldFragList ffl( String queryValue, String indexValue ) throws Exception { + make1d1fIndex( indexValue ); + Query query = paW.parse( queryValue ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + return new SimpleFragListBuilder().createFieldFragList( fpl, 20 ); + } + + public void test1PhraseShortMV() throws Exception { + makeIndexShortMV(); + + FieldQuery fq = new FieldQuery( tq( "d" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + assertEquals( "a b c d e", sfb.createFragment( reader, 0, F, ffl ) ); + } + + public void test1PhraseLongMV() throws Exception { + makeIndexLongMV(); + + FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + assertEquals( " most search engines use only one of these methods. Even the search engines that says they can use t", + sfb.createFragment( reader, 0, F, ffl ) ); + } + + public void test1PhraseLongMVB() throws Exception { + makeIndexLongMVB(); + + FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed" + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + SimpleFragmentsBuilder sfb = new SimpleFragmentsBuilder(); + assertEquals( "ssing speed, the", sfb.createFragment( reader, 0, F, ffl ) ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragmentsBuilderTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldQueryTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldQueryTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldQueryTest.java (revision 0) @@ -0,0 +1,822 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.highlight2.FieldQuery.QueryPhraseMap; +import org.apache.lucene.search.highlight2.FieldTermStack.TermInfo; + +public class FieldQueryTest extends AbstractTestCase { + + public void testFlattenBoolean() throws Exception { + Query query = paW.parse( "A AND B OR C NOT (D AND E)" ); + FieldQuery fq = new FieldQuery( query, true, true ); + Set flatQueries = new HashSet(); + fq.flatten( query, flatQueries ); + assertCollectionQueries( flatQueries, tq( "A" ), tq( "B" ), tq( "C" ) ); + } + + public void testFlattenTermAndPhrase() throws Exception { + Query query = paW.parse( "A AND \"B C\"" ); + FieldQuery fq = new FieldQuery( query, true, true ); + Set flatQueries = new HashSet(); + fq.flatten( query, flatQueries ); + assertCollectionQueries( flatQueries, tq( "A" ), pqF( "B", "C" ) ); + } + + public void testFlattenTermAndPhrase2gram() throws Exception { + Query query = paB.parse( "AA AND BCD OR EFGH" ); + FieldQuery fq = new FieldQuery( query, true, true ); + Set flatQueries = new HashSet(); + fq.flatten( query, flatQueries ); + assertCollectionQueries( flatQueries, tq( "AA" ), pqF( "BC", "CD" ), pqF( "EF", "FG", "GH" ) ); + } + + public void testFlatten1TermPhrase() throws Exception { + Query query = pqF( "A" ); + FieldQuery fq = new FieldQuery( query, true, true ); + Set flatQueries = new HashSet(); + fq.flatten( query, flatQueries ); + assertCollectionQueries( flatQueries, tq( "A" ) ); + } + + public void testExpand() throws Exception { + Query dummy = pqF( "DUMMY" ); + FieldQuery fq = new FieldQuery( dummy, true, true ); + + // "a b","b c" => "a b","b c","a b c" + Set flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "b", "c" ), pqF( "a", "b", "c" ) ); + + // "a b","b c d" => "a b","b c d","a b c d" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "b", "c", "d" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) ); + + // "a b c","b c d" => "a b c","b c d","a b c d" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b", "c" ) ); + flatQueries.add( pqF( "b", "c", "d" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b", "c" ), pqF( "b", "c", "d" ), pqF( "a", "b", "c", "d" ) ); + + // "a b c","c d e" => "a b c","c d e","a b c d e" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b", "c" ) ); + flatQueries.add( pqF( "c", "d", "e" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b", "c" ), pqF( "c", "d", "e" ), pqF( "a", "b", "c", "d", "e" ) ); + + // "a b b","b c" => "a b b","b c","a b b c" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b", "b" ) ); + flatQueries.add( pqF( "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b", "b" ), pqF( "b", "c" ), pqF( "a", "b", "b", "c" ) ); + + // "a b","b a" => "a b","b a","a b a", "b a b" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "b", "a" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "b", "a" ), pqF( "a", "b", "a" ), pqF( "b", "a", "b" ) ); + + // "a b","a b c" => "a b","a b c" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "a", "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "a", "b", "c" ) ); + } + + public void testNoExpand() throws Exception { + Query dummy = pqF( "DUMMY" ); + FieldQuery fq = new FieldQuery( dummy, true, true ); + + // "a b","c d" => "a b","c d" + Set flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "c", "d" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "c", "d" ) ); + + // "a","a b" => "a", "a b" + flatQueries = new HashSet(); + flatQueries.add( tq( "a" ) ); + flatQueries.add( pqF( "a", "b" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + tq( "a" ), pqF( "a", "b" ) ); + + // "a b","b" => "a b", "b" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( tq( "b" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), tq( "b" ) ); + + // "a b c","b c" => "a b c","b c" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b", "c" ) ); + flatQueries.add( pqF( "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b", "c" ), pqF( "b", "c" ) ); + + // "a b","a b c" => "a b","a b c" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b" ) ); + flatQueries.add( pqF( "a", "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b" ), pqF( "a", "b", "c" ) ); + + // "a b c","b d e" => "a b c","b d e" + flatQueries = new HashSet(); + flatQueries.add( pqF( "a", "b", "c" ) ); + flatQueries.add( pqF( "b", "d", "e" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pqF( "a", "b", "c" ), pqF( "b", "d", "e" ) ); + } + + public void testExpandNotFieldMatch() throws Exception { + Query dummy = pqF( "DUMMY" ); + FieldQuery fq = new FieldQuery( dummy, true, false ); + + // f1:"a b",f2:"b c" => f1:"a b",f2:"b c",f1:"a b c" + Set flatQueries = new HashSet(); + flatQueries.add( pq( F1, "a", "b" ) ); + flatQueries.add( pq( F2, "b", "c" ) ); + assertCollectionQueries( fq.expand( flatQueries ), + pq( F1, "a", "b" ), pq( F2, "b", "c" ), pq( F1, "a", "b", "c" ) ); + } + + public void testGetFieldTermMap() throws Exception { + Query query = tq( "a" ); + FieldQuery fq = new FieldQuery( query, true, true ); + + QueryPhraseMap pqm = fq.getFieldTermMap( F, "a" ); + assertNotNull( pqm ); + assertTrue( pqm.isTerminal() ); + + pqm = fq.getFieldTermMap( F, "b" ); + assertNull( pqm ); + + pqm = fq.getFieldTermMap( F1, "a" ); + assertNull( pqm ); + } + + public void testGetRootMap() throws Exception { + Query dummy = pqF( "DUMMY" ); + FieldQuery fq = new FieldQuery( dummy, true, true ); + + QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) ); + QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) ); + assertTrue( rootMap1 == rootMap2 ); + QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) ); + assertTrue( rootMap1 == rootMap3 ); + QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) ); + assertFalse( rootMap4 == rootMap3 ); + } + + public void testGetRootMapNotFieldMatch() throws Exception { + Query dummy = pqF( "DUMMY" ); + FieldQuery fq = new FieldQuery( dummy, true, false ); + + QueryPhraseMap rootMap1 = fq.getRootMap( tq( "a" ) ); + QueryPhraseMap rootMap2 = fq.getRootMap( tq( "a" ) ); + assertTrue( rootMap1 == rootMap2 ); + QueryPhraseMap rootMap3 = fq.getRootMap( tq( "b" ) ); + assertTrue( rootMap1 == rootMap3 ); + QueryPhraseMap rootMap4 = fq.getRootMap( tq( F1, "b" ) ); + assertTrue( rootMap4 == rootMap3 ); + } + + public void testGetTermSet() throws Exception { + Query query = paW.parse( "A AND B OR x:C NOT (D AND E)" ); + FieldQuery fq = new FieldQuery( query, true, true ); + assertEquals( 2, fq.termSetMap.size() ); + Set termSet = fq.getTermSet( F ); + assertEquals( 2, termSet.size() ); + assertTrue( termSet.contains( "A" ) ); + assertTrue( termSet.contains( "B" ) ); + termSet = fq.getTermSet( "x" ); + assertEquals( 1, termSet.size() ); + assertTrue( termSet.contains( "C" ) ); + termSet = fq.getTermSet( "y" ); + assertNull( termSet ); + } + + public void testQueryPhraseMap1Term() throws Exception { + Query query = tq( "a" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + assertTrue( qpm.subMap.get( "a" ) != null ); + assertTrue( qpm.subMap.get( "a" ).terminal ); + assertEquals( 1F, qpm.subMap.get( "a" ).boost ); + + // phraseHighlight = true, fieldMatch = false + fq = new FieldQuery( query, true, false ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( F ) ); + assertNotNull( map.get( null ) ); + qpm = map.get( null ); + assertEquals( 1, qpm.subMap.size() ); + assertTrue( qpm.subMap.get( "a" ) != null ); + assertTrue( qpm.subMap.get( "a" ).terminal ); + assertEquals( 1F, qpm.subMap.get( "a" ).boost ); + + // phraseHighlight = false, fieldMatch = true + fq = new FieldQuery( query, false, true ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + assertTrue( qpm.subMap.get( "a" ) != null ); + assertTrue( qpm.subMap.get( "a" ).terminal ); + assertEquals( 1F, qpm.subMap.get( "a" ).boost ); + + // phraseHighlight = false, fieldMatch = false + fq = new FieldQuery( query, false, false ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( F ) ); + assertNotNull( map.get( null ) ); + qpm = map.get( null ); + assertEquals( 1, qpm.subMap.size() ); + assertTrue( qpm.subMap.get( "a" ) != null ); + assertTrue( qpm.subMap.get( "a" ).terminal ); + assertEquals( 1F, qpm.subMap.get( "a" ).boost ); + + // boost != 1 + query = tq( 2, "a" ); + fq = new FieldQuery( query, true, true ); + map = fq.rootMaps; + qpm = map.get( F ); + assertEquals( 2F, qpm.subMap.get( "a" ).boost ); + } + + public void testQueryPhraseMap1Phrase() throws Exception { + Query query = pqF( "a", "b" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // phraseHighlight = true, fieldMatch = false + fq = new FieldQuery( query, true, false ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( F ) ); + assertNotNull( map.get( null ) ); + qpm = map.get( null ); + assertEquals( 1, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "a" ) ); + qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // phraseHighlight = false, fieldMatch = true + fq = new FieldQuery( query, false, true ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + qpm = map.get( F ); + assertEquals( 2, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "a" ) ); + qpm2 = qpm.subMap.get( "a" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + assertNotNull( qpm.subMap.get( "b" ) ); + qpm2 = qpm.subMap.get( "b" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + + // phraseHighlight = false, fieldMatch = false + fq = new FieldQuery( query, false, false ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( F ) ); + assertNotNull( map.get( null ) ); + qpm = map.get( null ); + assertEquals( 2, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "a" ) ); + qpm2 = qpm.subMap.get( "a" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + assertNotNull( qpm.subMap.get( "b" ) ); + qpm2 = qpm.subMap.get( "b" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + + // boost != 1 + query = pqF( 2, "a", "b" ); + // phraseHighlight = false, fieldMatch = false + fq = new FieldQuery( query, false, false ); + map = fq.rootMaps; + qpm = map.get( null ); + qpm2 = qpm.subMap.get( "a" ); + assertEquals( 2F, qpm2.boost ); + qpm3 = qpm2.subMap.get( "b" ); + assertEquals( 2F, qpm3.boost ); + qpm2 = qpm.subMap.get( "b" ); + assertEquals( 2F, qpm2.boost ); + } + + public void testQueryPhraseMap1PhraseAnother() throws Exception { + Query query = pqF( "search", "engines" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "search" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "search" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "engines" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "engines" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + } + + public void testQueryPhraseMap2Phrases() throws Exception { + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "b" ), Occur.SHOULD ); + query.add( pqF( 2, "c", "d" ), Occur.SHOULD ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 2, qpm.subMap.size() ); + + // "a b" + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "c d"^2 + assertNotNull( qpm.subMap.get( "c" ) ); + qpm2 = qpm.subMap.get( "c" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "d" ) ); + qpm3 = qpm2.subMap.get( "d" ); + assertTrue( qpm3.terminal ); + assertEquals( 2F, qpm3.boost ); + } + + public void testQueryPhraseMap2PhrasesFields() throws Exception { + BooleanQuery query = new BooleanQuery(); + query.add( pq( F1, "a", "b" ), Occur.SHOULD ); + query.add( pq( 2F, F2, "c", "d" ), Occur.SHOULD ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 2, map.size() ); + assertNull( map.get( null ) ); + + // "a b" + assertNotNull( map.get( F1 ) ); + QueryPhraseMap qpm = map.get( F1 ); + assertEquals( 1, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "c d"^2 + assertNotNull( map.get( F2 ) ); + qpm = map.get( F2 ); + assertEquals( 1, qpm.subMap.size() ); + assertNotNull( qpm.subMap.get( "c" ) ); + qpm2 = qpm.subMap.get( "c" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "d" ) ); + qpm3 = qpm2.subMap.get( "d" ); + assertTrue( qpm3.terminal ); + assertEquals( 2F, qpm3.boost ); + + // phraseHighlight = true, fieldMatch = false + fq = new FieldQuery( query, true, false ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( F1 ) ); + assertNull( map.get( F2 ) ); + assertNotNull( map.get( null ) ); + qpm = map.get( null ); + assertEquals( 2, qpm.subMap.size() ); + + // "a b" + assertNotNull( qpm.subMap.get( "a" ) ); + qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "c d"^2 + assertNotNull( qpm.subMap.get( "c" ) ); + qpm2 = qpm.subMap.get( "c" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "d" ) ); + qpm3 = qpm2.subMap.get( "d" ); + assertTrue( qpm3.terminal ); + assertEquals( 2F, qpm3.boost ); + } + + /* + * ...terminal + * + * a-b-c- + * +-d- + * b-c-d- + * +-d- + */ + public void testQueryPhraseMapOverlapPhrases() throws Exception { + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "b", "c" ), Occur.SHOULD ); + query.add( pqF( 2, "b", "c", "d" ), Occur.SHOULD ); + query.add( pqF( 3, "b", "d" ), Occur.SHOULD ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 2, qpm.subMap.size() ); + + // "a b c" + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "b" ); + assertFalse( qpm3.terminal ); + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "c" ) ); + QueryPhraseMap qpm4 = qpm3.subMap.get( "c" ); + assertTrue( qpm4.terminal ); + assertEquals( 1F, qpm4.boost ); + assertNotNull( qpm4.subMap.get( "d" ) ); + QueryPhraseMap qpm5 = qpm4.subMap.get( "d" ); + assertTrue( qpm5.terminal ); + assertEquals( 1F, qpm5.boost ); + + // "b c d"^2, "b d"^3 + assertNotNull( qpm.subMap.get( "b" ) ); + qpm2 = qpm.subMap.get( "b" ); + assertFalse( qpm2.terminal ); + assertEquals( 2, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "c" ) ); + qpm3 = qpm2.subMap.get( "c" ); + assertFalse( qpm3.terminal ); + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "d" ) ); + qpm4 = qpm3.subMap.get( "d" ); + assertTrue( qpm4.terminal ); + assertEquals( 2F, qpm4.boost ); + assertNotNull( qpm2.subMap.get( "d" ) ); + qpm3 = qpm2.subMap.get( "d" ); + assertTrue( qpm3.terminal ); + assertEquals( 3F, qpm3.boost ); + } + + /* + * ...terminal + * + * a-b- + * +-c- + */ + public void testQueryPhraseMapOverlapPhrases2() throws Exception { + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "b" ), Occur.SHOULD ); + query.add( pqF( 2, "a", "b", "c" ), Occur.SHOULD ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + + // "a b" + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "b" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "b" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "a b c"^2 + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "c" ) ); + QueryPhraseMap qpm4 = qpm3.subMap.get( "c" ); + assertTrue( qpm4.terminal ); + assertEquals( 2F, qpm4.boost ); + } + + /* + * ...terminal + * + * a-a-a- + * +-a- + * +-a- + * +-a- + */ + public void testQueryPhraseMapOverlapPhrases3() throws Exception { + BooleanQuery query = new BooleanQuery(); + query.add( pqF( "a", "a", "a", "a" ), Occur.SHOULD ); + query.add( pqF( 2, "a", "a", "a" ), Occur.SHOULD ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 1, qpm.subMap.size() ); + + // "a a a" + assertNotNull( qpm.subMap.get( "a" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "a" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "a" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "a" ); + assertFalse( qpm3.terminal ); + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "a" ) ); + QueryPhraseMap qpm4 = qpm3.subMap.get( "a" ); + assertTrue( qpm4.terminal ); + + // "a a a a" + assertEquals( 1, qpm4.subMap.size() ); + assertNotNull( qpm4.subMap.get( "a" ) ); + QueryPhraseMap qpm5 = qpm4.subMap.get( "a" ); + assertTrue( qpm5.terminal ); + + // "a a a a a" + assertEquals( 1, qpm5.subMap.size() ); + assertNotNull( qpm5.subMap.get( "a" ) ); + QueryPhraseMap qpm6 = qpm5.subMap.get( "a" ); + assertTrue( qpm6.terminal ); + + // "a a a a a a" + assertEquals( 1, qpm6.subMap.size() ); + assertNotNull( qpm6.subMap.get( "a" ) ); + QueryPhraseMap qpm7 = qpm6.subMap.get( "a" ); + assertTrue( qpm7.terminal ); + } + + public void testQueryPhraseMapOverlap2gram() throws Exception { + Query query = paB.parse( "abc AND bcd" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + Map map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + QueryPhraseMap qpm = map.get( F ); + assertEquals( 2, qpm.subMap.size() ); + + // "ab bc" + assertNotNull( qpm.subMap.get( "ab" ) ); + QueryPhraseMap qpm2 = qpm.subMap.get( "ab" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "bc" ) ); + QueryPhraseMap qpm3 = qpm2.subMap.get( "bc" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "ab bc cd" + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "cd" ) ); + QueryPhraseMap qpm4 = qpm3.subMap.get( "cd" ); + assertTrue( qpm4.terminal ); + assertEquals( 1F, qpm4.boost ); + + // "bc cd" + assertNotNull( qpm.subMap.get( "bc" ) ); + qpm2 = qpm.subMap.get( "bc" ); + assertFalse( qpm2.terminal ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "cd" ) ); + qpm3 = qpm2.subMap.get( "cd" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // phraseHighlight = false, fieldMatch = true + fq = new FieldQuery( query, false, true ); + map = fq.rootMaps; + assertEquals( 1, map.size() ); + assertNull( map.get( null ) ); + assertNotNull( map.get( F ) ); + qpm = map.get( F ); + assertEquals( 3, qpm.subMap.size() ); + + // "ab bc" + assertNotNull( qpm.subMap.get( "ab" ) ); + qpm2 = qpm.subMap.get( "ab" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "bc" ) ); + qpm3 = qpm2.subMap.get( "bc" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "ab bc cd" + assertEquals( 1, qpm3.subMap.size() ); + assertNotNull( qpm3.subMap.get( "cd" ) ); + qpm4 = qpm3.subMap.get( "cd" ); + assertTrue( qpm4.terminal ); + assertEquals( 1F, qpm4.boost ); + + // "bc cd" + assertNotNull( qpm.subMap.get( "bc" ) ); + qpm2 = qpm.subMap.get( "bc" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + assertEquals( 1, qpm2.subMap.size() ); + assertNotNull( qpm2.subMap.get( "cd" ) ); + qpm3 = qpm2.subMap.get( "cd" ); + assertTrue( qpm3.terminal ); + assertEquals( 1F, qpm3.boost ); + + // "cd" + assertNotNull( qpm.subMap.get( "cd" ) ); + qpm2 = qpm.subMap.get( "cd" ); + assertTrue( qpm2.terminal ); + assertEquals( 1F, qpm2.boost ); + assertEquals( 0, qpm2.subMap.size() ); + } + + public void testSearchPhrase() throws Exception { + Query query = pqF( "a", "b", "c" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + + // "a" + List phraseCandidate = new ArrayList(); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + assertNull( fq.searchPhrase( F, phraseCandidate ) ); + // "a b" + phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) ); + assertNull( fq.searchPhrase( F, phraseCandidate ) ); + // "a b c" + phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) ); + assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); + assertNull( fq.searchPhrase( "x", phraseCandidate ) ); + + // phraseHighlight = true, fieldMatch = false + fq = new FieldQuery( query, true, false ); + + // "a b c" + assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); + assertNotNull( fq.searchPhrase( "x", phraseCandidate ) ); + + // phraseHighlight = false, fieldMatch = true + fq = new FieldQuery( query, false, true ); + + // "a" + phraseCandidate.clear(); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); + // "a b" + phraseCandidate.add( new TermInfo( "b", 2, 3, 1 ) ); + assertNull( fq.searchPhrase( F, phraseCandidate ) ); + // "a b c" + phraseCandidate.add( new TermInfo( "c", 4, 5, 2 ) ); + assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); + assertNull( fq.searchPhrase( "x", phraseCandidate ) ); + } + + public void testSearchPhraseSlop() throws Exception { + // "a b c"~0 + Query query = pqF( "a", "b", "c" ); + + // phraseHighlight = true, fieldMatch = true + FieldQuery fq = new FieldQuery( query, true, true ); + + // "a b c" w/ position-gap = 2 + List phraseCandidate = new ArrayList(); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 2 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 4 ) ); + assertNull( fq.searchPhrase( F, phraseCandidate ) ); + + // "a b c"~1 + query = pqF( 1F, 1, "a", "b", "c" ); + + // phraseHighlight = true, fieldMatch = true + fq = new FieldQuery( query, true, true ); + + // "a b c" w/ position-gap = 2 + assertNotNull( fq.searchPhrase( F, phraseCandidate ) ); + + // "a b c" w/ position-gap = 3 + phraseCandidate.clear(); + phraseCandidate.add( new TermInfo( "a", 0, 1, 0 ) ); + phraseCandidate.add( new TermInfo( "b", 2, 3, 3 ) ); + phraseCandidate.add( new TermInfo( "c", 4, 5, 6 ) ); + assertNull( fq.searchPhrase( F, phraseCandidate ) ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/FieldQueryTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragListBuilderTest.java =================================================================== --- contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragListBuilderTest.java (revision 0) +++ contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragListBuilderTest.java (revision 0) @@ -0,0 +1,157 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Query; + +public class SimpleFragListBuilderTest extends AbstractTestCase { + + public void testNullFieldFragList() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "b c d" ), 100 ); + assertEquals( 0, ffl.fragInfos.size() ); + } + + public void testTooSmallFragSize() throws Exception { + try{ + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + sflb.createFieldFragList( fpl( "a", "b c d" ), SimpleFragListBuilder.MIN_FRAG_CHAR_SIZE - 1 ); + fail( "IllegalArgumentException must be thrown" ); + } + catch ( IllegalArgumentException expected ) { + } + } + + public void test1TermIndex() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a" ), 100 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1)))/1.0(0,100)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void test2TermsIndex1Frag() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a a" ), 100 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1))a((2,3)))/2.0(0,100)", ffl.fragInfos.get( 0 ).toString() ); + + ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b a" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1))a((18,19)))/2.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + + ffl = sflb.createFieldFragList( fpl( "a", "b b b b a b b b b a" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((8,9))a((18,19)))/2.0(2,22)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void test2TermsIndex2Frags() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b b b b b a" ), 20 ); + assertEquals( 2, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + assertEquals( "subInfos=(a((28,29)))/1.0(22,42)", ffl.fragInfos.get( 1 ).toString() ); + + ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b b b b a" ), 20 ); + assertEquals( 2, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + assertEquals( "subInfos=(a((26,27)))/1.0(20,40)", ffl.fragInfos.get( 1 ).toString() ); + + ffl = sflb.createFieldFragList( fpl( "a", "a b b b b b b b b b a" ), 20 ); + assertEquals( 2, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + assertEquals( "subInfos=(a((20,21)))/1.0(20,40)", ffl.fragInfos.get( 1 ).toString() ); + } + + public void test2TermsQuery() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "a b", "c d e" ), 20 ); + assertEquals( 0, ffl.fragInfos.size() ); + + ffl = sflb.createFieldFragList( fpl( "a b", "d b c" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(b((2,3)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + + ffl = sflb.createFieldFragList( fpl( "a b", "a b c" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(a((0,1))b((2,3)))/2.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void testPhraseQuery() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "\"a b\"", "c d e" ), 20 ); + assertEquals( 0, ffl.fragInfos.size() ); + + ffl = sflb.createFieldFragList( fpl( "\"a b\"", "a c b" ), 20 ); + assertEquals( 0, ffl.fragInfos.size() ); + + ffl = sflb.createFieldFragList( fpl( "\"a b\"", "a b c" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(ab((0,3)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void testPhraseQuerySlop() throws Exception { + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl( "\"a b\"~1", "a c b" ), 20 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(ab((0,1)(4,5)))/1.0(0,20)", ffl.fragInfos.get( 0 ).toString() ); + } + + private FieldPhraseList fpl( String queryValue, String indexValue ) throws Exception { + make1d1fIndex( indexValue ); + Query query = paW.parse( queryValue ); + FieldQuery fq = new FieldQuery( query, true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + return new FieldPhraseList( stack, fq ); + } + + public void test1PhraseShortMV() throws Exception { + makeIndexShortMV(); + + FieldQuery fq = new FieldQuery( tq( "d" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(d((6,7)))/1.0(0,100)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void test1PhraseLongMV() throws Exception { + makeIndexLongMV(); + + FieldQuery fq = new FieldQuery( pqF( "search", "engines" ), true, true ); + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(searchengines((102,116))searchengines((157,171)))/2.0(96,196)", ffl.fragInfos.get( 0 ).toString() ); + } + + public void test1PhraseLongMVB() throws Exception { + makeIndexLongMVB(); + + FieldQuery fq = new FieldQuery( pqF( "sp", "pe", "ee", "ed" ), true, true ); // "speed" -(2gram)-> "sp","pe","ee","ed" + FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); + FieldPhraseList fpl = new FieldPhraseList( stack, fq ); + SimpleFragListBuilder sflb = new SimpleFragListBuilder(); + FieldFragList ffl = sflb.createFieldFragList( fpl, 100 ); + assertEquals( 1, ffl.fragInfos.size() ); + assertEquals( "subInfos=(sppeeeed((88,93)))/1.0(82,182)", ffl.fragInfos.get( 0 ).toString() ); + } +} Property changes on: contrib/highlighter2/src/test/org/apache/lucene/search/highlight2/SimpleFragListBuilderTest.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldFragList.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldFragList.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldFragList.java (revision 0) @@ -0,0 +1,103 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.search.highlight2.FieldPhraseList.WeightedPhraseInfo; +import org.apache.lucene.search.highlight2.FieldPhraseList.WeightedPhraseInfo.Toffs; + +/** + * FieldFragList has a list of "frag info" that is used by FragmentsBuilder class + * to create fragments (snippets). + */ +public class FieldFragList { + + private final int fragCharSize; + List fragInfos = new ArrayList(); + + /** + * a constructor. + * + * @param fragCharSize the length (number of chars) of a fragment + */ + public FieldFragList( int fragCharSize ){ + this.fragCharSize = fragCharSize; + } + + /** + * convert the list of WeightedPhraseInfo to WeightedFragInfo, then add it to the fragInfos + * + * @param startOffset start offset of the fragment + * @param endOffset end offset of the fragment + * @param phraseInfoList list of WeightedPhraseInfo objects + */ + public void add( int startOffset, int endOffset, List phraseInfoList ){ + fragInfos.add( new WeightedFragInfo( startOffset, endOffset, phraseInfoList ) ); + } + + public static class WeightedFragInfo { + + List subInfos; + float totalBoost; + int startOffset; + int endOffset; + + public WeightedFragInfo( int startOffset, int endOffset, List phraseInfoList ){ + this.startOffset = startOffset; + this.endOffset = endOffset; + subInfos = new ArrayList(); + for( WeightedPhraseInfo phraseInfo : phraseInfoList ){ + SubInfo subInfo = new SubInfo( phraseInfo.text, phraseInfo.termsOffsets, phraseInfo.seqnum ); + subInfos.add( subInfo ); + totalBoost += phraseInfo.boost; + } + } + + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append( "subInfos=(" ); + for( SubInfo si : subInfos ) + sb.append( si.toString() ); + sb.append( ")/" ).append( totalBoost ).append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' ); + return sb.toString(); + } + + static class SubInfo { + final String text; // unnecessary member, just exists for debugging purpose + final List termsOffsets; // usually termsOffsets.size() == 1, + // but if position-gap > 1 and slop > 0 then size() could be greater than 1 + int seqnum; + SubInfo( String text, List termsOffsets, int seqnum ){ + this.text = text; + this.termsOffsets = termsOffsets; + this.seqnum = seqnum; + } + + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append( text ).append( '(' ); + for( Toffs to : termsOffsets ) + sb.append( to.toString() ); + sb.append( ')' ); + return sb.toString(); + } + } + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldFragList.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldTermStack.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldTermStack.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldTermStack.java (revision 0) @@ -0,0 +1,171 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collections; +import java.util.LinkedList; +import java.util.Set; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermPositionVector; +import org.apache.lucene.index.TermVectorOffsetInfo; +import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; + +/** + * FieldTermStack is a stack that keeps query terms in the specified field + * of the document to be highlighted. + */ +public class FieldTermStack { + + private final String fieldName; + LinkedList termList = new LinkedList(); + + public static void main( String[] args ) throws Exception { + Analyzer analyzer = new WhitespaceAnalyzer(); + QueryParser parser = new QueryParser( "f", analyzer ); + Query query = parser.parse( "a x:b" ); + FieldQuery fieldQuery = new FieldQuery( query, true, false ); + + Directory dir = new RAMDirectory(); + IndexWriter writer = new IndexWriter( dir, analyzer, MaxFieldLength.LIMITED ); + Document doc = new Document(); + doc.add( new Field( "f", "a a a b b c a b b c d e f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) ); + doc.add( new Field( "f", "b a b a f", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) ); + writer.addDocument( doc ); + writer.close(); + + IndexReader reader = IndexReader.open( dir ); + FieldTermStack ftl = new FieldTermStack( reader, 0, "f", fieldQuery ); + reader.close(); + } + + /** + * a constructor. + * + * @param reader IndexReader of the index + * @param docId document id to be highlighted + * @param fieldName field of the document to be highlighted + * @param fieldQuery FieldQuery object + * @throws IOException + */ + public FieldTermStack( IndexReader reader, int docId, String fieldName, final FieldQuery fieldQuery ) throws IOException { + this.fieldName = fieldName; + + TermFreqVector tfv = reader.getTermFreqVector( docId, fieldName ); + if( tfv == null ) return; // just return to make null snippets + TermPositionVector tpv = null; + try{ + tpv = (TermPositionVector)tfv; + } + catch( ClassCastException e ){ + return; // just return to make null snippets + } + + Set termSet = fieldQuery.getTermSet( fieldName ); + // just return to make null snippet if un-matched fieldName specified when fieldMatch == true + if( termSet == null ) return; + + for( String term : tpv.getTerms() ){ + if( !termSet.contains( term ) ) continue; + int index = tpv.indexOf( term ); + TermVectorOffsetInfo[] tvois = tpv.getOffsets( index ); + if( tvois == null ) return; // just return to make null snippets + int[] poss = tpv.getTermPositions( index ); + if( poss == null ) return; // just return to make null snippets + for( int i = 0; i < tvois.length; i++ ) + termList.add( new TermInfo( term, tvois[i].getStartOffset(), tvois[i].getEndOffset(), poss[i] ) ); + } + + // sort by position + Collections.sort( termList ); + } + + /** + * @return field name + */ + public String getFieldName(){ + return fieldName; + } + + /** + * @return the top TermInfo object of the stack + */ + public TermInfo pop(){ + return termList.poll(); + } + + /** + * @param termInfo the TermInfo object to be put on the top of the stack + */ + public void push( TermInfo termInfo ){ + // termList.push( termInfo ); // avoid Java 1.6 feature + termList.addFirst( termInfo ); + } + + /** + * to know whether the stack is empty + * + * @return true if the stack is empty, false if not + */ + public boolean isEmpty(){ + return termList == null || termList.size() == 0; + } + + public static class TermInfo implements Comparable{ + + final String text; + final int startOffset; + final int endOffset; + final int position; + + TermInfo( String text, int startOffset, int endOffset, int position ){ + this.text = text; + this.startOffset = startOffset; + this.endOffset = endOffset; + this.position = position; + } + + public String getText(){ return text; } + public int getStartOffset(){ return startOffset; } + public int getEndOffset(){ return endOffset; } + public int getPosition(){ return position; } + + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append( text ).append( '(' ).append(startOffset).append( ',' ).append( endOffset ).append( ',' ).append( position ).append( ')' ); + return sb.toString(); + } + + public int compareTo( TermInfo o ) { + return ( this.position - o.position ); + } + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldTermStack.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldPhraseList.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldPhraseList.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldPhraseList.java (revision 0) @@ -0,0 +1,183 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.apache.lucene.search.highlight2.FieldQuery.QueryPhraseMap; +import org.apache.lucene.search.highlight2.FieldTermStack.TermInfo; + +/** + * FieldPhraseList has a list of WeightedPhraseInfo that is used by FragListBuilder + * to create a FieldFragList object. + */ +public class FieldPhraseList { + + LinkedList phraseList = new LinkedList(); + + /** + * a constructor. + * + * @param fieldTermStack FieldTermStack object + * @param fieldQuery FieldQuery object + */ + public FieldPhraseList( FieldTermStack fieldTermStack, FieldQuery fieldQuery ){ + final String field = fieldTermStack.getFieldName(); + + LinkedList phraseCandidate = new LinkedList(); + QueryPhraseMap currMap = null; + QueryPhraseMap nextMap = null; + while( !fieldTermStack.isEmpty() ){ + + phraseCandidate.clear(); + + TermInfo ti = fieldTermStack.pop(); + currMap = fieldQuery.getFieldTermMap( field, ti.getText() ); + + // if not found, discard top TermInfo from stack, then try next element + if( currMap == null ) continue; + + // if found, search the longest phrase + phraseCandidate.add( ti ); + while( true ){ + ti = fieldTermStack.pop(); + nextMap = null; + if( ti != null ) + nextMap = currMap.getTermMap( ti.getText() ); + if( ti == null || nextMap == null ){ + if( ti != null ) + fieldTermStack.push( ti ); + if( currMap.isValidTermOrPhrase( phraseCandidate ) ){ + addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) ); + } + else{ + while( phraseCandidate.size() > 1 ){ + fieldTermStack.push( phraseCandidate.removeLast() ); + currMap = fieldQuery.searchPhrase( field, phraseCandidate ); + if( currMap != null ){ + addIfNoOverlap( new WeightedPhraseInfo( phraseCandidate, currMap.getBoost(), currMap.getTermOrPhraseNumber() ) ); + break; + } + } + } + break; + } + else{ + phraseCandidate.add( ti ); + currMap = nextMap; + } + } + } + } + + void addIfNoOverlap( WeightedPhraseInfo wpi ){ + for( WeightedPhraseInfo existWpi : phraseList ){ + if( existWpi.isOffsetOverlap( wpi ) ) return; + } + phraseList.add( wpi ); + } + + public static class WeightedPhraseInfo { + + String text; // unnecessary member, just exists for debugging purpose + List termsOffsets; // usually termsOffsets.size() == 1, + // but if position-gap > 1 and slop > 0 then size() could be greater than 1 + float boost; // query boost + int seqnum; + + public WeightedPhraseInfo( LinkedList terms, float boost ){ + this( terms, boost, 0 ); + } + + public WeightedPhraseInfo( LinkedList terms, float boost, int number ){ + this.boost = boost; + this.seqnum = number; + termsOffsets = new ArrayList( terms.size() ); + TermInfo ti = terms.get( 0 ); + termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) ); + if( terms.size() == 1 ){ + text = ti.getText(); + return; + } + StringBuilder sb = new StringBuilder(); + sb.append( ti.getText() ); + int pos = ti.getPosition(); + for( int i = 1; i < terms.size(); i++ ){ + ti = terms.get( i ); + sb.append( ti.getText() ); + if( ti.getPosition() - pos == 1 ){ + Toffs to = termsOffsets.get( termsOffsets.size() - 1 ); + to.setEndOffset( ti.getEndOffset() ); + } + else{ + termsOffsets.add( new Toffs( ti.getStartOffset(), ti.getEndOffset() ) ); + } + pos = ti.getPosition(); + } + text = sb.toString(); + } + + public int getStartOffset(){ + return termsOffsets.get( 0 ).startOffset; + } + + public int getEndOffset(){ + return termsOffsets.get( termsOffsets.size() - 1 ).endOffset; + } + + public boolean isOffsetOverlap( WeightedPhraseInfo other ){ + int so = getStartOffset(); + int eo = getEndOffset(); + int oso = other.getStartOffset(); + int oeo = other.getEndOffset(); + if( so <= oso && oso <= eo ) return true; + if( so <= oeo && oeo <= eo ) return true; + if( oso <= so && so <= oeo ) return true; + if( oso <= eo && eo <= oeo ) return true; + return false; + } + + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append( text ).append( '(' ).append( boost ).append( ")(" ); + for( Toffs to : termsOffsets ){ + sb.append( to ); + } + sb.append( ')' ); + return sb.toString(); + } + + public static class Toffs { + int startOffset; + int endOffset; + public Toffs( int startOffset, int endOffset ){ + this.startOffset = startOffset; + this.endOffset = endOffset; + } + void setEndOffset( int endOffset ){ + this.endOffset = endOffset; + } + public String toString(){ + StringBuilder sb = new StringBuilder(); + sb.append( '(' ).append( startOffset ).append( ',' ).append( endOffset ).append( ')' ); + return sb.toString(); + } + } + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldPhraseList.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragListBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragListBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragListBuilder.java (revision 0) @@ -0,0 +1,34 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * FragListBuilder is an interface for FieldFragList builder classes. + * A FragListBuilder class can be plugged in to Highlighter. + */ +public interface FragListBuilder { + + /** + * create a FieldFragList. + * + * @param fieldPhraseList FieldPhraseList object + * @param fragCharSize the length (number of chars) of a fragment + * @return the created FieldFragList object + */ + public FieldFragList createFieldFragList( FieldPhraseList fieldPhraseList, int fragCharSize ); +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragListBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilder.java (revision 0) @@ -0,0 +1,69 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import org.apache.lucene.search.highlight2.FieldFragList.WeightedFragInfo; + +/** + * An implementation of FragmentsBuilder that outputs score-order fragments. + */ +public class ScoreOrderFragmentsBuilder extends BaseFragmentsBuilder { + + /** + * a constructor. + */ + public ScoreOrderFragmentsBuilder(){ + super(); + } + + /** + * a constructor. + * + * @param preTags aray of pre-tags for markup terms. + * @param postTags array of post-tags for markup terms. + */ + public ScoreOrderFragmentsBuilder( String[] preTags, String[] postTags ){ + super( preTags, postTags ); + } + + /** + * Sort by score the list of WeightedFragInfo + */ + public List getWeightedFragInfoList( List src ) { + Collections.sort( src, new ScoreComparator() ); + return src; + } + + public static class ScoreComparator implements Comparator { + + public int compare( WeightedFragInfo o1, WeightedFragInfo o2 ) { + if( o1.totalBoost > o2.totalBoost ) return -1; + else if( o1.totalBoost < o2.totalBoost ) return 1; + // if same score then check startOffset + else{ + if( o1.startOffset < o2.startOffset ) return -1; + else if( o1.startOffset > o2.startOffset ) return 1; + } + return 0; + } + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/ScoreOrderFragmentsBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragmentsBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragmentsBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragmentsBuilder.java (revision 0) @@ -0,0 +1,53 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.List; + +import org.apache.lucene.search.highlight2.FieldFragList.WeightedFragInfo; + +/** + * A simple implementation of FragmentsBuilder. + * + */ +public class SimpleFragmentsBuilder extends BaseFragmentsBuilder { + + /** + * a constructor. + */ + public SimpleFragmentsBuilder() { + super(); + } + + /** + * a constructor. + * + * @param preTags array of pre-tags for markup terms. + * @param postTags array of post-tags for markup terms. + */ + public SimpleFragmentsBuilder( String[] preTags, String[] postTags ) { + super( preTags, postTags ); + } + + /** + * do nothing. return the source list. + */ + public List getWeightedFragInfoList( List src ) { + return src; + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragmentsBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/BaseFragmentsBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/BaseFragmentsBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/BaseFragmentsBuilder.java (revision 0) @@ -0,0 +1,124 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.MapFieldSelector; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.highlight2.FieldFragList.WeightedFragInfo; +import org.apache.lucene.search.highlight2.FieldFragList.WeightedFragInfo.SubInfo; +import org.apache.lucene.search.highlight2.FieldPhraseList.WeightedPhraseInfo.Toffs; + +public abstract class BaseFragmentsBuilder implements FragmentsBuilder { + + protected String[] preTags, postTags; + public static final String[] COLORED_PRE_TAGS = { + "", "", "", + "", "", "", + "", "", "", + "" + }; + public static final String[] COLORED_POST_TAGS = { "" }; + + protected BaseFragmentsBuilder(){ + this( new String[]{ "" }, new String[]{ "" } ); + } + + protected BaseFragmentsBuilder( String[] preTags, String[] postTags ){ + this.preTags = preTags; + this.postTags = postTags; + } + + static Object checkTagsArgument( Object tags ){ + if( tags instanceof String ) return tags; + else if( tags instanceof String[] ) return tags; + throw new IllegalArgumentException( "type of preTags/postTags must be a String or String[]" ); + } + + public abstract List getWeightedFragInfoList( List src ); + + public String createFragment( IndexReader reader, int docId, + String fieldName, FieldFragList fieldFragList ) throws IOException { + String[] fragments = createFragments( reader, docId, fieldName, fieldFragList, 1 ); + if( fragments == null || fragments.length == 0 ) return null; + return fragments[0]; + } + + public String[] createFragments( IndexReader reader, int docId, + String fieldName, FieldFragList fieldFragList, int maxNumFragments ) + throws IOException { + if( maxNumFragments < 0 ) + throw new IllegalArgumentException( "maxNumFragments(" + maxNumFragments + ") must be positive number." ); + + List fragInfos = getWeightedFragInfoList( fieldFragList.fragInfos ); + + List fragments = new ArrayList( maxNumFragments ); + String[] values = getFieldValues( reader, docId, fieldName ); + StringBuilder buffer = new StringBuilder(); + int[] nextValueIndex = { 0 }; + for( int n = 0; n < maxNumFragments && n < fragInfos.size(); n++ ){ + WeightedFragInfo fragInfo = fragInfos.get( n ); + fragments.add( makeFragment( buffer, nextValueIndex, values, fragInfo ) ); + } + return fragments.toArray( new String[fragments.size()] ); + } + + protected String[] getFieldValues( IndexReader reader, int docId, String fieldName) throws IOException { + Document doc = reader.document( docId, new MapFieldSelector( new String[]{ fieldName } ) ); + return doc.getValues( fieldName ); + } + + protected String makeFragment( StringBuilder buffer, int[] index, String[] values, WeightedFragInfo fragInfo ){ + StringBuilder fragment = new StringBuilder(); + final int s = fragInfo.startOffset; + String src = getFragmentSource( buffer, index, values, s, fragInfo.endOffset ); + int srcIndex = 0; + for( SubInfo subInfo : fragInfo.subInfos ){ + for( Toffs to : subInfo.termsOffsets ){ + fragment.append( src.substring( srcIndex, to.startOffset - s ) ).append( getPreTag( subInfo.seqnum ) ) + .append( src.substring( to.startOffset - s, to.endOffset - s ) ).append( getPostTag( subInfo.seqnum ) ); + srcIndex = to.endOffset - s; + } + } + fragment.append( src.substring( srcIndex ) ); + return fragment.toString(); + } + + protected String getFragmentSource( StringBuilder buffer, int[] index, String[] values, + int startOffset, int endOffset ){ + while( buffer.length() < endOffset && index[0] < values.length ){ + if( index[0] > 0 && values[index[0]].length() > 0 ) + buffer.append( ' ' ); + buffer.append( values[index[0]++] ); + } + int eo = buffer.length() < endOffset ? buffer.length() : endOffset; + return buffer.substring( startOffset, eo ); + } + + protected String getPreTag( int num ){ + return preTags.length > num ? preTags[num] : preTags[0]; + } + + protected String getPostTag( int num ){ + return postTags.length > num ? postTags[num] : postTags[0]; + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/BaseFragmentsBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldQuery.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldQuery.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldQuery.java (revision 0) @@ -0,0 +1,391 @@ +package org.apache.lucene.search.highlight2; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.highlight2.FieldTermStack.TermInfo; + +/** + * FieldQuery breaks down query object into terms/phrases and keep + * them in QueryPhraseMap structure. + */ +public class FieldQuery { + + final boolean fieldMatch; + + // fieldMatch==true, Map + // fieldMatch==false, Map + Map rootMaps = new HashMap(); + + // fieldMatch==true, Map + // fieldMatch==false, Map + Map> termSetMap = new HashMap>(); + + int termOrPhraseNumber; // used for colored tag support + + FieldQuery( Query query, boolean phraseHighlight, boolean fieldMatch ){ + this.fieldMatch = fieldMatch; + Set flatQueries = new HashSet(); + flatten( query, flatQueries ); + saveTerms( flatQueries ); + Collection expandQueries = expand( flatQueries ); + + for( Query flatQuery : expandQueries ){ + QueryPhraseMap rootMap = getRootMap( flatQuery ); + rootMap.add( flatQuery ); + if( !phraseHighlight && flatQuery instanceof PhraseQuery ){ + PhraseQuery pq = (PhraseQuery)flatQuery; + if( pq.getTerms().length > 1 ){ + for( Term term : pq.getTerms() ) + rootMap.addTerm( term, flatQuery.getBoost() ); + } + } + } + } + + void flatten( Query sourceQuery, Collection flatQueries ){ + if( sourceQuery instanceof BooleanQuery ){ + BooleanQuery bq = (BooleanQuery)sourceQuery; + for( BooleanClause clause : bq.getClauses() ){ + if( !clause.isProhibited() ) + flatten( clause.getQuery(), flatQueries ); + } + } + else if( sourceQuery instanceof TermQuery ){ + if( !flatQueries.contains( sourceQuery ) ) + flatQueries.add( sourceQuery ); + } + else if( sourceQuery instanceof PhraseQuery ){ + if( !flatQueries.contains( sourceQuery ) ){ + PhraseQuery pq = (PhraseQuery)sourceQuery; + if( pq.getTerms().length > 1 ) + flatQueries.add( pq ); + else if( pq.getTerms().length == 1 ){ + flatQueries.add( new TermQuery( pq.getTerms()[0] ) ); + } + } + } + // else discard queries + } + + /* + * Create expandQueries from flatQueries. + * + * expandQueries := flatQueries + overlapped phrase queries + * + * ex1) flatQueries={a,b,c} + * => expandQueries={a,b,c} + * ex2) flatQueries={a,"b c","c d"} + * => expandQueries={a,"b c","c d","b c d"} + */ + Collection expand( Collection flatQueries ){ + Set expandQueries = new HashSet(); + for( Iterator i = flatQueries.iterator(); i.hasNext(); ){ + Query query = i.next(); + i.remove(); + expandQueries.add( query ); + if( !( query instanceof PhraseQuery ) ) continue; + for( Iterator j = flatQueries.iterator(); j.hasNext(); ){ + Query qj = j.next(); + if( !( qj instanceof PhraseQuery ) ) continue; + checkOverlap( expandQueries, (PhraseQuery)query, (PhraseQuery)qj ); + } + } + return expandQueries; + } + + /* + * Check if PhraseQuery A and B have overlapped part. + * + * ex1) A="a b", B="b c" => overlap; expandQueries={"a b c"} + * ex2) A="b c", B="a b" => overlap; expandQueries={"a b c"} + * ex3) A="a b", B="c d" => no overlap; expandQueries={} + */ + private void checkOverlap( Collection expandQueries, PhraseQuery a, PhraseQuery b ){ + if( a.getSlop() != b.getSlop() ) return; + Term[] ats = a.getTerms(); + Term[] bts = b.getTerms(); + if( fieldMatch && !ats[0].field().equals( bts[0].field() ) ) return; + checkOverlap( expandQueries, ats, bts, a.getSlop(), a.getBoost() ); + checkOverlap( expandQueries, bts, ats, b.getSlop(), b.getBoost() ); + } + + /* + * Check if src and dest have overlapped part and if it is, create PhraseQueries and add expandQueries. + * + * ex1) src="a b", dest="c d" => no overlap + * ex2) src="a b", dest="a b c" => no overlap + * ex3) src="a b", dest="b c" => overlap; expandQueries={"a b c"} + * ex4) src="a b c", dest="b c d" => overlap; expandQueries={"a b c d"} + * ex5) src="a b c", dest="b c" => no overlap + * ex6) src="a b c", dest="b" => no overlap + * ex7) src="a a a a", dest="a a a" => overlap; + * expandQueries={"a a a a a","a a a a a a"} + */ + private void checkOverlap( Collection expandQueries, Term[] src, Term[] dest, int slop, float boost ){ + // beginning from 1 (not 0) is safe because that the PhraseQuery has multiple terms + // is guaranteed in flatten() method (if PhraseQuery has only one term, flatten() + // converts PhraseQuery to TermQuery) + for( int i = 1; i < src.length; i++ ){ + boolean overlap = true; + for( int j = i; j < src.length; j++ ){ + if( !src[j].text().equals( dest[j-i].text() ) ){ + overlap = false; + break; + } + } + if( overlap && src.length - i < dest.length ){ + PhraseQuery pq = new PhraseQuery(); + for( Term srcTerm : src ) + pq.add( srcTerm ); + for( int k = src.length - i; k < dest.length; k++ ){ + pq.add( new Term( src[0].field(), dest[k].text() ) ); + } + pq.setSlop( slop ); + pq.setBoost( boost ); + if(!expandQueries.contains( pq ) ) + expandQueries.add( pq ); + } + } + } + + QueryPhraseMap getRootMap( Query query ){ + String key = getKey( query ); + QueryPhraseMap map = rootMaps.get( key ); + if( map == null ){ + map = new QueryPhraseMap( this ); + rootMaps.put( key, map ); + } + return map; + } + + /* + * Return 'key' string. 'key' is the field name of the Query. + * If not fieldMatch, 'key' will be null. + */ + private String getKey( Query query ){ + if( !fieldMatch ) return null; + if( query instanceof TermQuery ) + return ((TermQuery)query).getTerm().field(); + else if ( query instanceof PhraseQuery ){ + PhraseQuery pq = (PhraseQuery)query; + Term[] terms = pq.getTerms(); + return terms[0].field(); + } + else + throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); + } + + /* + * Save the set of terms in the queries to termSetMap. + * + * ex1) q=name:john + * - fieldMatch==true + * termSetMap=Map<"name",Set<"john">> + * - fieldMatch==false + * termSetMap=Map> + * + * ex2) q=name:john title:manager + * - fieldMatch==true + * termSetMap=Map<"name",Set<"john">, + * "title",Set<"manager">> + * - fieldMatch==false + * termSetMap=Map> + * + * ex3) q=name:"john lennon" + * - fieldMatch==true + * termSetMap=Map<"name",Set<"john","lennon">> + * - fieldMatch==false + * termSetMap=Map> + */ + void saveTerms( Collection flatQueries ){ + for( Query query : flatQueries ){ + Set termSet = getTermSet( query ); + if( query instanceof TermQuery ) + termSet.add( ((TermQuery)query).getTerm().text() ); + else if( query instanceof PhraseQuery ){ + for( Term term : ((PhraseQuery)query).getTerms() ) + termSet.add( term.text() ); + } + else + throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); + } + } + + private Set getTermSet( Query query ){ + String key = getKey( query ); + Set set = termSetMap.get( key ); + if( set == null ){ + set = new HashSet(); + termSetMap.put( key, set ); + } + return set; + } + + Set getTermSet( String field ){ + return termSetMap.get( fieldMatch ? field : null ); + } + + /** + * + * @param fieldName + * @param term + * @return + */ + public QueryPhraseMap getFieldTermMap( String fieldName, String term ){ + QueryPhraseMap rootMap = getRootMap( fieldName ); + return rootMap == null ? null : rootMap.subMap.get( term ); + } + + /** + * + * @param fieldName + * @param phraseCandidate + * @return + */ + public QueryPhraseMap searchPhrase( String fieldName, final List phraseCandidate ){ + QueryPhraseMap root = getRootMap( fieldName ); + if( root == null ) return null; + return root.searchPhrase( phraseCandidate ); + } + + private QueryPhraseMap getRootMap( String fieldName ){ + return rootMaps.get( fieldMatch ? fieldName : null ); + } + + int nextTermOrPhraseNumber(){ + return termOrPhraseNumber++; + } + + public static class QueryPhraseMap { + + boolean terminal; + int slop; // valid if terminal == true and phraseHighlight == true + float boost; // valid if terminal == true + int termOrPhraseNumber; // valid if terminal == true + FieldQuery fieldQuery; + Map subMap = new HashMap(); + + public QueryPhraseMap( FieldQuery fieldQuery ){ + this.fieldQuery = fieldQuery; + } + + void addTerm( Term term, float boost ){ + QueryPhraseMap map = getOrNewMap( subMap, term.text() ); + map.markTerminal( boost ); + } + + private QueryPhraseMap getOrNewMap( Map subMap, String term ){ + QueryPhraseMap map = subMap.get( term ); + if( map == null ){ + map = new QueryPhraseMap( fieldQuery ); + subMap.put( term, map ); + } + return map; + } + + void add( Query query ){ + if( query instanceof TermQuery ){ + addTerm( ((TermQuery)query).getTerm(), query.getBoost() ); + } + else if( query instanceof PhraseQuery ){ + PhraseQuery pq = (PhraseQuery)query; + Term[] terms = pq.getTerms(); + Map map = subMap; + QueryPhraseMap qpm = null; + for( Term term : terms ){ + qpm = getOrNewMap( map, term.text() ); + map = qpm.subMap; + } + qpm.markTerminal( pq.getSlop(), pq.getBoost() ); + } + else + throw new RuntimeException( "query \"" + query.toString() + "\" must be flatten first." ); + } + + public QueryPhraseMap getTermMap( String term ){ + return subMap.get( term ); + } + + private void markTerminal( float boost ){ + markTerminal( 0, boost ); + } + + private void markTerminal( int slop, float boost ){ + this.terminal = true; + this.slop = slop; + this.boost = boost; + this.termOrPhraseNumber = fieldQuery.nextTermOrPhraseNumber(); + } + + public boolean isTerminal(){ + return terminal; + } + + public int getSlop(){ + return slop; + } + + public float getBoost(){ + return boost; + } + + public int getTermOrPhraseNumber(){ + return termOrPhraseNumber; + } + + public QueryPhraseMap searchPhrase( final List phraseCandidate ){ + QueryPhraseMap currMap = this; + for( TermInfo ti : phraseCandidate ){ + currMap = currMap.subMap.get( ti.getText() ); + if( currMap == null ) return null; + } + return currMap.isValidTermOrPhrase( phraseCandidate ) ? currMap : null; + } + + public boolean isValidTermOrPhrase( final List phraseCandidate ){ + // check terminal + if( !terminal ) return false; + + // if the candidate is a term, it is valid + if( phraseCandidate.size() == 1 ) return true; + + // else check whether the candidate is valid phrase + // compare position-gaps between terms to slop + int pos = phraseCandidate.get( 0 ).getPosition(); + for( int i = 1; i < phraseCandidate.size(); i++ ){ + int nextPos = phraseCandidate.get( i ).getPosition(); + if( Math.abs( nextPos - pos - 1 ) > slop ) return false; + pos = nextPos; + } + return true; + } + } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FieldQuery.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragListBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragListBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragListBuilder.java (revision 0) @@ -0,0 +1,82 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.search.highlight2.FieldPhraseList.WeightedPhraseInfo; + +/** + * A simple implementation of FragListBuilder. + */ +public class SimpleFragListBuilder implements FragListBuilder { + + public static final int MARGIN = 6; + public static final int MIN_FRAG_CHAR_SIZE = MARGIN * 3; + + public FieldFragList createFieldFragList(FieldPhraseList fieldPhraseList, int fragCharSize) { + if( fragCharSize < MIN_FRAG_CHAR_SIZE ) + throw new IllegalArgumentException( "fragCharSize(" + fragCharSize + ") is too small. It must be " + + MIN_FRAG_CHAR_SIZE + " or higher." ); + + FieldFragList ffl = new FieldFragList( fragCharSize ); + + List wpil = new ArrayList(); + Iterator ite = fieldPhraseList.phraseList.iterator(); + WeightedPhraseInfo phraseInfo = null; + int startOffset = 0; + boolean taken = false; + while( true ){ + if( !taken ){ + if( !ite.hasNext() ) break; + phraseInfo = ite.next(); + } + taken = false; + if( phraseInfo == null ) break; + + // if the phrase violates the border of previous fragment, discard it and try next phrase + if( phraseInfo.getStartOffset() < startOffset ) continue; + + wpil.clear(); + wpil.add( phraseInfo ); + int st = phraseInfo.getStartOffset() - MARGIN < startOffset ? + startOffset : phraseInfo.getStartOffset() - MARGIN; + int en = st + fragCharSize; + startOffset = en; + + while( true ){ + if( ite.hasNext() ){ + phraseInfo = ite.next(); + taken = true; + if( phraseInfo == null ) break; + } + else + break; + if( phraseInfo.getEndOffset() <= en ) + wpil.add( phraseInfo ); + else + break; + } + ffl.add( st, en, wpil ); + } + return ffl; + } + +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/SimpleFragListBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragmentsBuilder.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragmentsBuilder.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragmentsBuilder.java (revision 0) @@ -0,0 +1,57 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; + +/** + * FragmentsBuilder is an interface for fragments (snippets) builder classes. + * A FragmentsBuilder class can be plugged in to Highlighter. + */ +public interface FragmentsBuilder { + + /** + * create a fragment. + * + * @param reader IndexReader of the index + * @param docId document id to be highlighted + * @param fieldName field of the document to be highlighted + * @param fieldFragList FieldFragList object + * @return a created fragment or null when no fragment created + * @throws IOException + */ + public String createFragment( IndexReader reader, int docId, String fieldName, + FieldFragList fieldFragList ) throws IOException; + + /** + * create multiple fragments. + * + * @param reader IndexReader of the index + * @param docId document id to be highlighter + * @param fieldName field of the document to be highlighted + * @param fieldFragList FieldFragList object + * @param maxNumFragments maximum number of fragments + * @return created fragments or null when no fragments created. + * size of the array can be less than maxNumFragments + * @throws IOException + */ + public String[] createFragments( IndexReader reader, int docId, String fieldName, + FieldFragList fieldFragList, int maxNumFragments ) throws IOException; +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/FragmentsBuilder.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/Highlighter.java =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/Highlighter.java (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/Highlighter.java (revision 0) @@ -0,0 +1,137 @@ +package org.apache.lucene.search.highlight2; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Query; + +/** + * Another highlighter implementation. + * + */ +public class Highlighter { + + public static final boolean DEFAULT_PHRASE_HIGHLIGHT = true; + public static final boolean DEFAULT_FIELD_MATCH = true; + private final boolean phraseHighlight; + private final boolean fieldMatch; + private final FragListBuilder fragListBuilder; + private final FragmentsBuilder fragmentsBuilder; + + /** + * the default constructor. + */ + public Highlighter(){ + this( DEFAULT_PHRASE_HIGHLIGHT, DEFAULT_FIELD_MATCH ); + } + + /** + * a constructor. Using SimpleFragListBuilder and ScoreOrderFragmentsBuilder. + * + * @param phraseHighlight true or false for phrase highlighting + * @param fieldMatch true of false for field matching + */ + public Highlighter( boolean phraseHighlight, boolean fieldMatch ){ + this( phraseHighlight, fieldMatch, new SimpleFragListBuilder(), new ScoreOrderFragmentsBuilder() ); + } + + /** + * a constructor. A FragListBuilder and a FragmentsBuilder can be specified (plugins). + * + * @param phraseHighlight true of false for phrase highlighting + * @param fieldMatch true of false for field matching + * @param fragListBuilder an instance of FragListBuilder + * @param fragmentsBuilder an instance of FragmentsBuilder + */ + public Highlighter( boolean phraseHighlight, boolean fieldMatch, + FragListBuilder fragListBuilder, FragmentsBuilder fragmentsBuilder ){ + this.phraseHighlight = phraseHighlight; + this.fieldMatch = fieldMatch; + this.fragListBuilder = fragListBuilder; + this.fragmentsBuilder = fragmentsBuilder; + } + + /** + * create a FieldQuery object. + * + * @param query a query + * @return the created FieldQuery object + */ + public FieldQuery getFieldQuery( Query query ){ + return new FieldQuery( query, phraseHighlight, fieldMatch ); + } + + /** + * return the best fragment. + * + * @param fieldQuery FieldQuery object + * @param reader IndexReader of the index + * @param docId document id to be highlighted + * @param fieldName field of the document to be highlighted + * @param fragCharSize the length (number of chars) of a fragment + * @return the best fragment (snippet) string + * @throws IOException + */ + public final String getBestFragment( final FieldQuery fieldQuery, IndexReader reader, int docId, + String fieldName, int fragCharSize ) throws IOException { + FieldFragList fieldFragList = getFieldFragList( fieldQuery, reader, docId, fieldName, fragCharSize ); + return fragmentsBuilder.createFragment( reader, docId, fieldName, fieldFragList ); + } + + /** + * return the best fragments. + * + * @param fieldQuery FieldQuery object + * @param reader IndexReader of the index + * @param docId document id to be highlighted + * @param fieldName field of the document to be highlighted + * @param fragCharSize the length (number of chars) of a fragment + * @param maxNumFragments maximum number of fragments + * @return created fragments or null when no fragments created. + * size of the array can be less than maxNumFragments + * @throws IOException + */ + public final String[] getBestFragments( final FieldQuery fieldQuery, IndexReader reader, int docId, + String fieldName, int fragCharSize, int maxNumFragments ) throws IOException { + FieldFragList fieldFragList = getFieldFragList( fieldQuery, reader, docId, fieldName, fragCharSize ); + return fragmentsBuilder.createFragments( reader, docId, fieldName, fieldFragList, maxNumFragments ); + } + + private FieldFragList getFieldFragList( final FieldQuery fieldQuery, IndexReader reader, int docId, + String fieldName, int fragCharSize ) throws IOException { + FieldTermStack fieldTermStack = new FieldTermStack( reader, docId, fieldName, fieldQuery ); + FieldPhraseList fieldPhraseList = new FieldPhraseList( fieldTermStack, fieldQuery ); + return fragListBuilder.createFieldFragList( fieldPhraseList, fragCharSize ); + } + + /** + * return whether phraseHighlight or not. + * + * @return + */ + public boolean isPhraseHighlight(){ return phraseHighlight; } + + /** + * return whether fieldMatch or not. + * + * @return + */ + public boolean isFieldMatch(){ return fieldMatch; } +} Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/Highlighter.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/package.html =================================================================== --- contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/package.html (revision 0) +++ contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/package.html (revision 0) @@ -0,0 +1,126 @@ + + +This is an another highlighter implementation. + +

Features

+
    +
  • fast for large docs
    • +
  • support N-gram fields
    • +
  • support phrase-unit highlighting with slops
    • +
  • need Java 1.5
    • +
  • highlight fields need to be TermVector.WITH_POSITIONS_OFFSETS
    • +
  • take into account query boost to score fragments
    • +
  • support colored highlight tags
    • +
  • pluggable FragListBuilder
    • +
  • pluggable FragmentsBuilder
    • +
+ +

Algorithm

+

To explain the algorithm, let's use the following sample text + (to be highlighted) and user query:

+ + + + + + + + + + +
Sample TextLucene is a search engine library.
User QueryLucene^2 OR "search library"~1
+ +

The user query is a BooleanQuery that consists of TermQuery("Lucene") +with boost of 2 and PhraseQuery("search library") with slop of 1.

+

For your convenience, here is the offsets and positions info of the +sample text.

+ +
++--------+-----------------------------------+
+|        |          1111111111222222222233333|
+|  offset|01234567890123456789012345678901234|
++--------+-----------------------------------+
+|document|Lucene is a search engine library. |
++--------*-----------------------------------+
+|position|0      1  2 3      4      5        |
++--------*-----------------------------------+
+
+ +

Step 1.

+

In Step 1, H2 generates {@link org.apache.lucene.search.highlight2.FieldQuery.QueryPhraseMap} from the user query. +QueryPhraseMap consists of the following members:

+
+public class QueryPhraseMap {
+  boolean terminal;
+  int slop;   // valid if terminal == true and phraseHighlight == true
+  float boost;  // valid if terminal == true
+  Map<String, QueryPhraseMap> subMap;
+} 
+
+

QueryPhraseMap has subMap. The key of the subMap is a term +text in the user query and the value is a subsequent QueryPhraseMap. +If the query is a term (not phrase), then the subsequent QueryPhraseMap +is marked as terminal. If the query is a phrase, then the subsequent QueryPhraseMap +is not a terminal and it has the next term text in the phrase.

+ +

From the sample user query, the following QueryPhraseMap +will be generated:

+
+   QueryPhraseMap
++--------+-+  +-------+-+
+|"Lucene"|o+->|boost=2|*|  * : terminal
++--------+-+  +-------+-+
+
++--------+-+  +---------+-+  +-------+------+-+
+|"search"|o+->|"library"|o+->|boost=1|slop=1|*|
++--------+-+  +---------+-+  +-------+------+-+
+
+ +

Step 2.

+

In Step 2, H2 generates {@link org.apache.lucene.search.highlight2.FieldTermStack}. H2 uses {@link org.apache.lucene.index.TermFreqVector} data +(must be stored {@link org.apache.lucene.document.Field.TermVector#WITH_POSITIONS_OFFSETS}) +to generate it. FieldTermStack keeps the terms in the user query. +Therefore, in this sample case, H2 generates the following FieldTermStack:

+
+   FieldTermStack
++------------------+
+|"Lucene"(0,6,0)   |
++------------------+
+|"search"(12,18,3) |
++------------------+
+|"library"(26,33,5)|
++------------------+
+where : "termText"(startOffset,endOffset,position)
+
+

Step 3.

+

In Step 3, H2 generates {@link org.apache.lucene.search.highlight2.FieldPhraseList} +by reference to QueryPhraseMap and FieldTermStack.

+
+   FieldPhraseList
++----------------+-----------------+---+
+|"Lucene"        |[(0,6)]          |w=2|
++----------------+-----------------+---+
+|"search library"|[(12,18),(26,33)]|w=1|
++----------------+-----------------+---+
+
+

The type of each entry is WeightedPhraseInfo that consists of +an array of terms offsets and weight. The weight (H2 uses query boost to +calculate the weight) will be taken into account when H2 creates +{@link org.apache.lucene.search.highlight2.FieldFragList} in the next step.

+

Step 4.

+

In Step 4, H2 creates FieldFragList by reference to +FieldPhraseList. In this sample case, the following +FieldFragList will be generated:

+
+   FieldFragList
++---------------------------------+
+|"Lucene"[(0,6)]                  |
+|"search library"[(12,18),(26,33)]|
+|totalBoost=3                     |
++---------------------------------+
+
+

Step 5.

+

In Step 5, by using FieldFragList and the field stored data, +H2 creates highlighted snippets!

+ + Property changes on: contrib/highlighter2/src/java/org/apache/lucene/search/highlight2/package.html ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/highlighter2/build.xml =================================================================== --- contrib/highlighter2/build.xml (revision 0) +++ contrib/highlighter2/build.xml (revision 0) @@ -0,0 +1,47 @@ + + + + + + + + Hits highlighter2 + + + + + + + + + + + + + + + + + + + + Highlighter2 building dependency ${analyzers.jar} + + + + Property changes on: contrib/highlighter2/build.xml ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native