Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java
===================================================================
--- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (revision 799886)
+++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (working copy)
@@ -38,7 +38,7 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.highlight.Highlighter;
-import org.apache.lucene.search.highlight.QueryScorer;
+import org.apache.lucene.search.highlight.QueryTermScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.search.highlight.TextFragment;
import org.apache.lucene.search.highlight.TokenSources;
@@ -242,7 +242,7 @@
}
protected Highlighter getHighlighter(Query q){
- return new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(q));
+ return new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(q));
}
/**
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (working copy)
@@ -1,161 +1,263 @@
package org.apache.lucene.search.highlight;
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
+import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Query;
+
/**
* {@link Scorer} implementation which scores text fragments by the number of
- * unique query terms found. This class uses the {@link QueryTermExtractor}
- * class to process determine the query terms and their boosts to be used.
+ * unique query terms found. This class converts appropriate Querys to
+ * SpanQuerys and attempts to score only those terms that participated in
+ * generating the 'hit' on the document.
*/
-// TODO: provide option to boost score of fragments near beginning of document
-// based on fragment.getFragNum()
public class QueryScorer implements Scorer {
-
- TextFragment currentTextFragment = null;
- HashSet uniqueTermsInFragment;
+ private float totalScore;
+ private Set foundTerms;
+ private Map fieldWeightedSpanTerms;
+ private float maxTermWeight;
+ private int position = -1;
+ private String defaultField;
+ private TermAttribute termAtt;
+ private PositionIncrementAttribute posIncAtt;
+ private boolean expandMultiTermQuery;
+ private Query query;
+ private String field;
+ private IndexReader reader;
+ private static boolean highlightCnstScrRngQuery;
- float totalScore = 0;
- float maxTermWeight = 0;
- private HashMap termsToFind;
+ /**
+ * @param query
+ * Query to use for highlighting
+ * @param field
+ * Field to highlight - pass null to ignore fields
+ * @param tokenStream
+ * of source text to be highlighted
+ * @throws IOException
+ */
+ public QueryScorer(Query query, String field) {
+ init(query, field, null, true);
+ }
+
- private TermAttribute termAtt;
+ /**
+ * @param query
+ * Query to use for highlighting
+ * @param field
+ * Field to highlight - pass null to ignore fields
+ * @param tokenStream
+ * of source text to be highlighted
+ * @param expandMultiTermQuery
+ * rewrite multi-term queries against a single doc memory index to
+ * create boolean queries
+ * @throws IOException
+ */
+ public QueryScorer(Query query, String field, boolean expandMultiTermQuery) {
+ init(query, field, null, expandMultiTermQuery);
+ }
/**
- *
- * @param query a Lucene query (ideally rewritten using query.rewrite before
- * being passed to this class and the searcher)
+ * @param query
+ * Query to use for highlighting
+ * @param field
+ * Field to highlight - pass null to ignore fields
+ * @param tokenStream
+ * of source text to be highlighted
+ * @param reader
+ * @throws IOException
+ */
+ public QueryScorer(Query query, String field, IndexReader reader) {
+ init(query, field, reader, true);
+ }
+
+ /**
+ * @param query
+ * Query to use for highlighting
+ * @param field
+ * Field to highlight - pass null to ignore fields
+ * @param tokenStream
+ * of source text to be highlighted
+ * @param reader
+ * @param expandMultiTermQuery
+ * rewrite multi-term queries against a single doc memory index to
+ * create boolean queries
+ * @throws IOException
*/
- public QueryScorer(Query query) {
- this(QueryTermExtractor.getTerms(query));
+ public QueryScorer(Query query, String field, IndexReader reader, boolean expandMultiTermQuery) {
+ init(query, field, reader, expandMultiTermQuery);
}
/**
- *
- * @param query a Lucene query (ideally rewritten using query.rewrite before
- * being passed to this class and the searcher)
- * @param fieldName the Field name which is used to match Query terms
+ * As above, but with ability to pass in an IndexReader
+ */
+ public QueryScorer(Query query, String field, IndexReader reader, String defaultField)
+ throws IOException {
+ this.defaultField = defaultField.intern();
+ init(query, field, reader, true);
+ }
+
+ /**
+ * As above, but with ability to pass in an IndexReader
*/
- public QueryScorer(Query query, String fieldName) {
- this(QueryTermExtractor.getTerms(query, false, fieldName));
+ public QueryScorer(Query query, String field, IndexReader reader, String defaultField, boolean expandMultiTermQuery) {
+ this.defaultField = defaultField.intern();
+ init(query, field, reader, expandMultiTermQuery);
}
/**
- *
- * @param query a Lucene query (ideally rewritten using query.rewrite before
- * being passed to this class and the searcher)
- * @param reader used to compute IDF which can be used to a) score selected
- * fragments better b) use graded highlights eg set font color
- * intensity
- * @param fieldName the field on which Inverse Document Frequency (IDF)
- * calculations are based
+ * @param defaultField - The default field for queries with the field name unspecified
+ */
+ public QueryScorer(Query query, String field, String defaultField) {
+ this.defaultField = defaultField.intern();
+ init(query, field, null, true);
+ }
+
+ /**
+ * @param defaultField - The default field for queries with the field name unspecified
*/
- public QueryScorer(Query query, IndexReader reader, String fieldName) {
- this(QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName));
+ public QueryScorer(Query query, String field, String defaultField, boolean expandMultiTermQuery) {
+ this.defaultField = defaultField.intern();
+ init(query, field, null, expandMultiTermQuery);
}
- public QueryScorer(WeightedTerm[] weightedTerms) {
- termsToFind = new HashMap();
+ /**
+ * @param weightedTerms
+ */
+ public QueryScorer(WeightedSpanTerm[] weightedTerms) {
+ this.fieldWeightedSpanTerms = new HashMap(weightedTerms.length);
+
for (int i = 0; i < weightedTerms.length; i++) {
- WeightedTerm existingTerm = (WeightedTerm) termsToFind
- .get(weightedTerms[i].term);
- if ((existingTerm == null)
- || (existingTerm.weight < weightedTerms[i].weight)) {
- // if a term is defined more than once, always use the highest scoring
- // weight
- termsToFind.put(weightedTerms[i].term, weightedTerms[i]);
+ WeightedSpanTerm existingTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(weightedTerms[i].term);
+
+ if ((existingTerm == null) ||
+ (existingTerm.weight < weightedTerms[i].weight)) {
+ // if a term is defined more than once, always use the highest
+ // scoring weight
+ fieldWeightedSpanTerms.put(weightedTerms[i].term, weightedTerms[i]);
maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
}
}
}
- /* (non-Javadoc)
- * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
- */
- public void init(TokenStream tokenStream) {
- termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
- }
-
/*
* (non-Javadoc)
- *
- * @see
- * org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
- * .lucene.search.highlight.TextFragment)
+ *
+ * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
*/
- public void startFragment(TextFragment newFragment) {
- uniqueTermsInFragment = new HashSet();
- currentTextFragment = newFragment;
- totalScore = 0;
-
+ public float getFragmentScore() {
+ return totalScore;
}
+ /**
+ *
+ * @return The highest weighted term (useful for passing to
+ * GradientFormatter to set top end of coloring scale.
+ */
+ public float getMaxTermWeight() {
+ return maxTermWeight;
+ }
- /* (non-Javadoc)
- * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.highlight.Scorer#getTokenScore(org.apache.lucene.analysis.Token,
+ * int)
*/
public float getTokenScore() {
+ position += posIncAtt.getPositionIncrement();
String termText = termAtt.term();
- WeightedTerm queryTerm = (WeightedTerm) termsToFind.get(termText);
- if (queryTerm == null) {
- // not a query term - return
+ WeightedSpanTerm weightedSpanTerm;
+
+ if ((weightedSpanTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(
+ termText)) == null) {
+ return 0;
+ }
+
+ if (weightedSpanTerm.positionSensitive &&
+ !weightedSpanTerm.checkPosition(position)) {
return 0;
}
+
+ float score = weightedSpanTerm.getWeight();
+
// found a query term - is it unique in this doc?
- if (!uniqueTermsInFragment.contains(termText)) {
- totalScore += queryTerm.getWeight();
- uniqueTermsInFragment.add(termText);
+ if (!foundTerms.contains(termText)) {
+ totalScore += score;
+ foundTerms.add(termText);
}
- return queryTerm.getWeight();
- }
+ return score;
+ }
- /* (non-Javadoc)
- * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
+ public void init(TokenStream tokenStream) {
+ termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+ posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
+ }
+
+ /**
+ * Retrieve the WeightedSpanTerm for the specified token. Useful for passing
+ * Span information to a Fragmenter.
+ *
+ * @param token
+ * @return WeightedSpanTerm for token
*/
- public float getFragmentScore() {
- return totalScore;
+ public WeightedSpanTerm getWeightedSpanTerm(String token) {
+ return (WeightedSpanTerm) fieldWeightedSpanTerms.get(token);
}
- /*
- * (non-Javadoc)
- *
- * @see
- * org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+ /**
+ * @param query
+ * @param field
+ * @param tokenStream
+ * @param reader
+ * @throws IOException
*/
- public void allFragmentsProcessed() {
- // this class has no special operations to perform at end of processing
+ private void init(Query query, String field, IndexReader reader, boolean expandMultiTermQuery) {
+ this.reader = reader;
+ this.expandMultiTermQuery = expandMultiTermQuery;
+ this.query = query;
+ this.field = field;
+ }
+
+ public void setTokenStream(CachingTokenFilter cachingTokenFilter) throws IOException {
+ WeightedSpanTermExtractor qse = defaultField == null ? new WeightedSpanTermExtractor()
+ : new WeightedSpanTermExtractor(defaultField);
+
+ qse.setExpandMultiTermQuery(expandMultiTermQuery);
+ if (reader == null) {
+ this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query,
+ cachingTokenFilter, field);
+ } else {
+ this.fieldWeightedSpanTerms = qse.getWeightedSpanTermsWithScores(query,
+ cachingTokenFilter, field, reader);
+ }
}
/**
- *
- * @return The highest weighted term (useful for passing to GradientFormatter
- * to set top end of coloring scale.
+ * If you call Highlighter#getBestFragment() more than once you must reset
+ * the SpanScorer between each call.
*/
- public float getMaxTermWeight() {
- return maxTermWeight;
+ public void reset() {
+ position = -1;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.search.highlight.Scorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
+ */
+ public void startFragment(TextFragment newFragment) {
+ foundTerms = new HashSet();
+ totalScore = 0;
}
}
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (revision 0)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (revision 0)
@@ -0,0 +1,161 @@
+package org.apache.lucene.search.highlight;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+
+/**
+ * {@link Scorer} implementation which scores text fragments by the number of
+ * unique query terms found. This class uses the {@link QueryTermExtractor}
+ * class to process determine the query terms and their boosts to be used.
+ */
+// TODO: provide option to boost score of fragments near beginning of document
+// based on fragment.getFragNum()
+public class QueryTermScorer implements Scorer {
+
+ TextFragment currentTextFragment = null;
+ HashSet uniqueTermsInFragment;
+
+ float totalScore = 0;
+ float maxTermWeight = 0;
+ private HashMap termsToFind;
+
+ private TermAttribute termAtt;
+
+ /**
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ */
+ public QueryTermScorer(Query query) {
+ this(QueryTermExtractor.getTerms(query));
+ }
+
+ /**
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ * @param fieldName the Field name which is used to match Query terms
+ */
+ public QueryTermScorer(Query query, String fieldName) {
+ this(QueryTermExtractor.getTerms(query, false, fieldName));
+ }
+
+ /**
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ * @param reader used to compute IDF which can be used to a) score selected
+ * fragments better b) use graded highlights eg set font color
+ * intensity
+ * @param fieldName the field on which Inverse Document Frequency (IDF)
+ * calculations are based
+ */
+ public QueryTermScorer(Query query, IndexReader reader, String fieldName) {
+ this(QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName));
+ }
+
+ public QueryTermScorer(WeightedTerm[] weightedTerms) {
+ termsToFind = new HashMap();
+ for (int i = 0; i < weightedTerms.length; i++) {
+ WeightedTerm existingTerm = (WeightedTerm) termsToFind
+ .get(weightedTerms[i].term);
+ if ((existingTerm == null)
+ || (existingTerm.weight < weightedTerms[i].weight)) {
+ // if a term is defined more than once, always use the highest scoring
+ // weight
+ termsToFind.put(weightedTerms[i].term, weightedTerms[i]);
+ maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
+ }
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
+ */
+ public void init(TokenStream tokenStream) {
+ termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
+ * .lucene.search.highlight.TextFragment)
+ */
+ public void startFragment(TextFragment newFragment) {
+ uniqueTermsInFragment = new HashSet();
+ currentTextFragment = newFragment;
+ totalScore = 0;
+
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
+ */
+ public float getTokenScore() {
+ String termText = termAtt.term();
+
+ WeightedTerm queryTerm = (WeightedTerm) termsToFind.get(termText);
+ if (queryTerm == null) {
+ // not a query term - return
+ return 0;
+ }
+ // found a query term - is it unique in this doc?
+ if (!uniqueTermsInFragment.contains(termText)) {
+ totalScore += queryTerm.getWeight();
+ uniqueTermsInFragment.add(termText);
+ }
+ return queryTerm.getWeight();
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
+ */
+ public float getFragmentScore() {
+ return totalScore;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+ */
+ public void allFragmentsProcessed() {
+ // this class has no special operations to perform at end of processing
+ }
+
+ /**
+ *
+ * @return The highest weighted term (useful for passing to GradientFormatter
+ * to set top end of coloring scale.
+ */
+ public float getMaxTermWeight() {
+ return maxTermWeight;
+ }
+}
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (working copy)
@@ -34,7 +34,7 @@
private int fragmentSize;
private int currentNumFrags;
private int position = -1;
- private SpanScorer spanScorer;
+ private QueryScorer QueryScorer;
private int waitForPos = -1;
private int textSize;
private TermAttribute termAtt;
@@ -42,19 +42,19 @@
private OffsetAttribute offsetAtt;
/**
- * @param spanscorer SpanScorer that was used to score hits
+ * @param QueryScorer QueryScorer that was used to score hits
*/
- public SimpleSpanFragmenter(SpanScorer spanscorer) {
- this(spanscorer, DEFAULT_FRAGMENT_SIZE);
+ public SimpleSpanFragmenter(QueryScorer QueryScorer) {
+ this(QueryScorer, DEFAULT_FRAGMENT_SIZE);
}
/**
- * @param spanscorer SpanScorer that was used to score hits
+ * @param QueryScorer QueryScorer that was used to score hits
* @param fragmentSize size in bytes of each fragment
*/
- public SimpleSpanFragmenter(SpanScorer spanscorer, int fragmentSize) {
+ public SimpleSpanFragmenter(QueryScorer QueryScorer, int fragmentSize) {
this.fragmentSize = fragmentSize;
- this.spanScorer = spanscorer;
+ this.QueryScorer = QueryScorer;
}
/* (non-Javadoc)
@@ -69,7 +69,7 @@
return false;
}
- WeightedSpanTerm wSpanTerm = spanScorer.getWeightedSpanTerm(termAtt.term());
+ WeightedSpanTerm wSpanTerm = QueryScorer.getWeightedSpanTerm(termAtt.term());
if (wSpanTerm != null) {
List positionSpans = wSpanTerm.getPositionSpans();
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java (revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java (working copy)
@@ -1,288 +0,0 @@
-package org.apache.lucene.search.highlight;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.Query;
-
-
-/**
- * {@link Scorer} implementation which scores text fragments by the number of
- * unique query terms found. This class converts appropriate Querys to
- * SpanQuerys and attempts to score only those terms that participated in
- * generating the 'hit' on the document.
- */
-public class SpanScorer implements Scorer {
- private float totalScore;
- private Set foundTerms;
- private Map fieldWeightedSpanTerms;
- private float maxTermWeight;
- private int position = -1;
- private String defaultField;
- private TermAttribute termAtt;
- private PositionIncrementAttribute posIncAtt;
- private static boolean highlightCnstScrRngQuery;
-
- /**
- * @param query
- * Query to use for highlighting
- * @param field
- * Field to highlight - pass null to ignore fields
- * @param tokenStream
- * of source text to be highlighted
- * @throws IOException
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter) throws IOException {
- init(query, field, cachingTokenFilter, null, false);
- }
-
-
- /**
- * @param query
- * Query to use for highlighting
- * @param field
- * Field to highlight - pass null to ignore fields
- * @param tokenStream
- * of source text to be highlighted
- * @param expandMultiTermQuery
- * rewrite multi-term queries against a single doc memory index to
- * create boolean queries
- * @throws IOException
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, boolean expandMultiTermQuery) throws IOException {
- init(query, field, cachingTokenFilter, null, expandMultiTermQuery);
- }
-
- /**
- * @param query
- * Query to use for highlighting
- * @param field
- * Field to highlight - pass null to ignore fields
- * @param tokenStream
- * of source text to be highlighted
- * @param reader
- * @throws IOException
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, IndexReader reader)
- throws IOException {
- init(query, field, cachingTokenFilter, reader, false);
- }
-
- /**
- * @param query
- * Query to use for highlighting
- * @param field
- * Field to highlight - pass null to ignore fields
- * @param tokenStream
- * of source text to be highlighted
- * @param reader
- * @param expandMultiTermQuery
- * rewrite multi-term queries against a single doc memory index to
- * create boolean queries
- * @throws IOException
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, IndexReader reader, boolean expandMultiTermQuery)
- throws IOException {
- init(query, field, cachingTokenFilter, reader, expandMultiTermQuery);
- }
-
- /**
- * As above, but with ability to pass in an IndexReader
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, IndexReader reader, String defaultField)
- throws IOException {
- this.defaultField = defaultField.intern();
- init(query, field, cachingTokenFilter, reader, false);
- }
-
- /**
- * As above, but with ability to pass in an IndexReader
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, IndexReader reader, String defaultField, boolean expandMultiTermQuery)
- throws IOException {
- this.defaultField = defaultField.intern();
- init(query, field, cachingTokenFilter, reader, expandMultiTermQuery);
- }
-
- /**
- * @param defaultField - The default field for queries with the field name unspecified
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, String defaultField) throws IOException {
- this.defaultField = defaultField.intern();
- init(query, field, cachingTokenFilter, null, false);
- }
-
- /**
- * @param defaultField - The default field for queries with the field name unspecified
- */
- public SpanScorer(Query query, String field,
- CachingTokenFilter cachingTokenFilter, String defaultField, boolean expandMultiTermQuery) throws IOException {
- this.defaultField = defaultField.intern();
- init(query, field, cachingTokenFilter, null, expandMultiTermQuery);
- }
-
- /**
- * @param weightedTerms
- */
- public SpanScorer(WeightedSpanTerm[] weightedTerms) {
- this.fieldWeightedSpanTerms = new HashMap(weightedTerms.length);
-
- for (int i = 0; i < weightedTerms.length; i++) {
- WeightedSpanTerm existingTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(weightedTerms[i].term);
-
- if ((existingTerm == null) ||
- (existingTerm.weight < weightedTerms[i].weight)) {
- // if a term is defined more than once, always use the highest
- // scoring weight
- fieldWeightedSpanTerms.put(weightedTerms[i].term, weightedTerms[i]);
- maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
- }
- }
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
- */
- public float getFragmentScore() {
- return totalScore;
- }
-
- /**
- *
- * @return The highest weighted term (useful for passing to
- * GradientFormatter to set top end of coloring scale.
- */
- public float getMaxTermWeight() {
- return maxTermWeight;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.search.highlight.Scorer#getTokenScore(org.apache.lucene.analysis.Token,
- * int)
- */
- public float getTokenScore() {
- position += posIncAtt.getPositionIncrement();
- String termText = termAtt.term();
-
- WeightedSpanTerm weightedSpanTerm;
-
- if ((weightedSpanTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(
- termText)) == null) {
- return 0;
- }
-
- if (weightedSpanTerm.positionSensitive &&
- !weightedSpanTerm.checkPosition(position)) {
- return 0;
- }
-
- float score = weightedSpanTerm.getWeight();
-
- // found a query term - is it unique in this doc?
- if (!foundTerms.contains(termText)) {
- totalScore += score;
- foundTerms.add(termText);
- }
-
- return score;
- }
-
- public void init(TokenStream tokenStream) {
- termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
- posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
- }
-
- /**
- * Retrieve the WeightedSpanTerm for the specified token. Useful for passing
- * Span information to a Fragmenter.
- *
- * @param token
- * @return WeightedSpanTerm for token
- */
- public WeightedSpanTerm getWeightedSpanTerm(String token) {
- return (WeightedSpanTerm) fieldWeightedSpanTerms.get(token);
- }
-
- /**
- * @param query
- * @param field
- * @param tokenStream
- * @param reader
- * @throws IOException
- */
- private void init(Query query, String field,
- CachingTokenFilter cachingTokenFilter, IndexReader reader, boolean expandMultiTermQuery)
- throws IOException {
- WeightedSpanTermExtractor qse = defaultField == null ? new WeightedSpanTermExtractor()
- : new WeightedSpanTermExtractor(defaultField);
-
- qse.setHighlightCnstScrRngQuery(highlightCnstScrRngQuery);
- qse.setExpandMultiTermQuery(expandMultiTermQuery);
- if (reader == null) {
- this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query,
- cachingTokenFilter, field);
- } else {
- this.fieldWeightedSpanTerms = qse.getWeightedSpanTermsWithScores(query,
- cachingTokenFilter, field, reader);
- }
- }
-
- /**
- * @return whether ConstantScoreRangeQuerys are set to be highlighted
- * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
- * constructor option to expand MultiTerm queries.
- */
- public static boolean isHighlightCnstScrRngQuery() {
- return highlightCnstScrRngQuery;
- }
-
- /**
- * If you call Highlighter#getBestFragment() more than once you must reset
- * the SpanScorer between each call.
- */
- public void reset() {
- position = -1;
- }
-
- /**
- * Turns highlighting of ConstantScoreRangeQuery on/off.
- * ConstantScoreRangeQuerys cannot be highlighted if you rewrite the query
- * first. Must be called before SpanScorer construction.
- *
- * @param highlightCnstScrRngQuery
- * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
- * constructor option to expand MultiTerm queries.
- */
- public static void setHighlightCnstScrRngQuery(boolean highlight) {
- highlightCnstScrRngQuery = highlight;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.search.highlight.Scorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
- */
- public void startFragment(TextFragment newFragment) {
- foundTerms = new HashSet();
- totalScore = 0;
- }
-}
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (working copy)
@@ -60,7 +60,6 @@
private CachingTokenFilter cachedTokenFilter;
private Map readers = new HashMap(10); // Map
private String defaultField;
- private boolean highlightCnstScrRngQuery;
private boolean expandMultiTermQuery;
public WeightedSpanTermExtractor() {
@@ -131,7 +130,7 @@
for (Iterator iterator = ((DisjunctionMaxQuery) query).iterator(); iterator.hasNext();) {
extract((Query) iterator.next(), terms);
}
- } else if (query instanceof MultiTermQuery && (highlightCnstScrRngQuery || expandMultiTermQuery)) {
+ } else if (query instanceof MultiTermQuery && expandMultiTermQuery) {
MultiTermQuery mtq = ((MultiTermQuery)query);
if(mtq.getRewriteMethod() != MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE) {
mtq = copyMultiTermQuery(mtq);
@@ -240,8 +239,7 @@
while (spans.next()) {
spanPositions.add(new PositionSpan(spans.start(), spans.end() - 1));
}
-
- cachedTokenFilter.reset();
+
}
if (spanPositions.size() == 0) {
@@ -301,15 +299,17 @@
return rv;
}
- private IndexReader getReaderForField(String field) {
+ private IndexReader getReaderForField(String field) throws IOException {
IndexReader reader = (IndexReader) readers.get(field);
if (reader == null) {
MemoryIndex indexer = new MemoryIndex();
indexer.addField(field, cachedTokenFilter);
+ cachedTokenFilter.reset();
IndexSearcher searcher = indexer.createSearcher();
reader = searcher.getIndexReader();
readers.put(field, reader);
}
+
return reader;
}
@@ -419,23 +419,6 @@
return terms;
}
-
- /**
- * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use
- * getExpandMultiTermQuery instead.
- */
- public boolean isHighlightCnstScrRngQuery() {
- return highlightCnstScrRngQuery;
- }
-
- /**
- * @param highlightCnstScrRngQuery
- * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
- * setExpandMultiTermQuery option.
- */
- public void setHighlightCnstScrRngQuery(boolean highlightCnstScrRngQuery) {
- this.highlightCnstScrRngQuery = highlightCnstScrRngQuery;
- }
/**
* This class makes sure that if both position sensitive and insensitive
Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
===================================================================
--- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 799886)
+++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy)
@@ -118,8 +118,9 @@
query = qp.parse("\"very long\"");
searcher = new IndexSearcher(ramDir, false);
TopDocs hits = searcher.search(query, 10);
-
- Highlighter highlighter = new Highlighter(null);
+
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(scorer);
for (int i = 0; i < hits.scoreDocs.length; i++) {
@@ -129,10 +130,10 @@
TokenStream stream = TokenSources.getAnyTokenStream(searcher
.getIndexReader(), hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
CachingTokenFilter ctf = new CachingTokenFilter(stream);
- SpanScorer scorer = new SpanScorer(query, FIELD_NAME, ctf);
- // ctf.reset();
+
+ scorer.setTokenStream(ctf);
Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
- highlighter.setFragmentScorer(scorer);
+
highlighter.setTextFragmenter(fragmenter);
String fragment = highlighter.getBestFragment(ctf, storedField);
@@ -181,10 +182,11 @@
fieldName, new StringReader(text)));
// Assuming "", "" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
- Highlighter highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName,
- tokenStream, FIELD_NAME));
+ QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(formatter, scorer);
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
- tokenStream.reset();
+
String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");
return rv.length() == 0 ? text : rv;
}
@@ -194,13 +196,15 @@
int maxNumFragmentsRequired = 2;
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(scorer);
+
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(new SpanScorer(query, FIELD_NAME, tokenStream));
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
- tokenStream.reset();
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -227,7 +231,8 @@
Query query = qp.parse(q);
CachingTokenFilter stream = new CachingTokenFilter(analyzer.tokenStream(f1,
new StringReader(content)));
- Scorer scorer = new SpanScorer(query, f1, stream, false);
+ QueryScorer scorer = new QueryScorer(query, f1, false);
+ scorer.setTokenStream(stream);
Highlighter h = new Highlighter(this, scorer);
h.getBestFragment(analyzer, f1, content);
@@ -241,14 +246,16 @@
int maxNumFragmentsRequired = 2;
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this, scorer);
+
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(this,
- new SpanScorer(query, FIELD_NAME, tokenStream));
+
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
- tokenStream.reset();
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -268,10 +275,10 @@
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(this,
- new SpanScorer(query, FIELD_NAME, tokenStream));
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this,scorer);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
- tokenStream.reset();
+ scorer.setTokenStream(tokenStream);
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -291,10 +298,10 @@
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(this,
- new SpanScorer(query, FIELD_NAME, tokenStream));
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this, scorer);
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
- tokenStream.reset();
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -310,14 +317,15 @@
int maxNumFragmentsRequired = 2;
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this, scorer);
+
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
.tokenStream(FIELD_NAME, new StringReader(text)));
- SpanScorer spanscorer = new SpanScorer(query, FIELD_NAME, tokenStream);
- Highlighter highlighter = new Highlighter(this, spanscorer);
- highlighter.setTextFragmenter(new SimpleSpanFragmenter(spanscorer, 5));
- tokenStream.reset();
+ scorer.setTokenStream(tokenStream);
+ highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
String result = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired, "...");
@@ -328,14 +336,17 @@
doSearching("\"been shot\"");
maxNumFragmentsRequired = 2;
+
+ scorer = new QueryScorer(query, FIELD_NAME);
+ highlighter = new Highlighter(this, scorer);
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
.tokenStream(FIELD_NAME, new StringReader(text)));
- SpanScorer spanscorer = new SpanScorer(query, FIELD_NAME, tokenStream);
- Highlighter highlighter = new Highlighter(this, spanscorer);
- highlighter.setTextFragmenter(new SimpleSpanFragmenter(spanscorer, 20));
+
+ scorer.setTokenStream(tokenStream);
+ highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
tokenStream.reset();
String result = highlighter.getBestFragments(tokenStream, text,
@@ -350,13 +361,16 @@
doSearching("y \"x y z\"");
int maxNumFragmentsRequired = 2;
-
+
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this,scorer);
+
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(this,
- new SpanScorer(query, FIELD_NAME, tokenStream));
+
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
tokenStream.reset();
@@ -421,7 +435,7 @@
public void testSimpleHighlighter() throws Exception {
doSearching("Kennedy");
- Highlighter highlighter = new Highlighter(new QueryScorer(query));
+ Highlighter highlighter = new Highlighter(new QueryTermScorer(query));
highlighter.setTextFragmenter(new SimpleFragmenter(40));
int maxNumFragmentsRequired = 2;
for (int i = 0; i < hits.length(); i++) {
@@ -579,15 +593,14 @@
String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
- SpanScorer scorer = null;
+ QueryScorer scorer = null;
TokenStream tokenStream = null;
tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
new StringReader(text)));
- SpanScorer.setHighlightCnstScrRngQuery(true);
- scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME, (CachingTokenFilter) tokenStream);
-
+ scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
+ scorer.setTokenStream((CachingTokenFilter) tokenStream);
Highlighter highlighter = new Highlighter(this, scorer);
((CachingTokenFilter) tokenStream).reset();
@@ -619,14 +632,14 @@
String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
- SpanScorer scorer = null;
+ QueryScorer scorer = null;
TokenStream tokenStream = null;
tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
new StringReader(text)));
- scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME, (CachingTokenFilter) tokenStream, true);
-
+ scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME, true);
+ scorer.setTokenStream((CachingTokenFilter) tokenStream);
Highlighter highlighter = new Highlighter(this, scorer);
((CachingTokenFilter) tokenStream).reset();
@@ -650,14 +663,14 @@
String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
- SpanScorer scorer = null;
+ QueryScorer scorer = null;
TokenStream tokenStream = null;
tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
new StringReader(text)));
- scorer = new SpanScorer(query, null, (CachingTokenFilter) tokenStream, true);
-
+ scorer = new QueryScorer(query, null, true);
+ scorer.setTokenStream((CachingTokenFilter) tokenStream);
Highlighter highlighter = new Highlighter(this, scorer);
((CachingTokenFilter) tokenStream).reset();
@@ -681,14 +694,14 @@
String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
int maxNumFragmentsRequired = 2;
String fragmentSeparator = "...";
- SpanScorer scorer = null;
+ QueryScorer scorer = null;
TokenStream tokenStream = null;
tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
new StringReader(text)));
- scorer = new SpanScorer(query, "random_field", (CachingTokenFilter) tokenStream, HighlighterTest.FIELD_NAME, true);
-
+ scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME, true);
+ scorer.setTokenStream((CachingTokenFilter) tokenStream);
Highlighter highlighter = new Highlighter(this, scorer);
((CachingTokenFilter) tokenStream).reset();
@@ -744,7 +757,7 @@
public void run() throws Exception {
TermQuery query = new TermQuery(new Term("data", "help"));
- Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));
+ Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(query));
hg.setTextFragmenter(new NullFragmenter());
String match = null;
@@ -900,7 +913,7 @@
Highlighter highlighter = getHighlighter(wTerms, HighlighterTest.this);// new
// Highlighter(new
- // QueryScorer(wTerms));
+ // QueryTermScorer(wTerms));
TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
highlighter.setTextFragmenter(new SimpleFragmenter(2));
@@ -965,7 +978,7 @@
public void run() throws Exception {
numHighlights = 0;
doSearching("Kennedy");
- // new Highlighter(HighlighterTest.this, new QueryScorer(query));
+ // new Highlighter(HighlighterTest.this, new QueryTermScorer(query));
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
@@ -995,7 +1008,7 @@
Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
HighlighterTest.this);// new Highlighter(this, new
- // QueryScorer(query));
+ // QueryTermScorer(query));
highlighter.setTextFragmenter(new SimpleFragmenter(20));
String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);
@@ -1027,7 +1040,7 @@
TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
HighlighterTest.this);// new Highlighter(this, new
- // QueryScorer(query));
+ // QueryTermScorer(query));
highlighter.setMaxDocBytesToAnalyze(30);
highlighter.getBestFragment(tokenStream, texts[0]);
@@ -1062,7 +1075,7 @@
Highlighter hg = getHighlighter(query, "data", new StandardAnalyzer(stopWords).tokenStream(
"data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
// new
- // QueryScorer(query));
+ // QueryTermScorer(query));
hg.setTextFragmenter(new NullFragmenter());
hg.setMaxDocBytesToAnalyze(100);
match = hg.getBestFragment(new StandardAnalyzer(stopWords), "data", sb.toString());
@@ -1114,7 +1127,6 @@
public void run() throws Exception {
numHighlights = 0;
- SpanScorer.setHighlightCnstScrRngQuery(false);
// test to show how rewritten query can still be used
searcher = new IndexSearcher(ramDir);
Analyzer analyzer = new StandardAnalyzer();
@@ -1136,12 +1148,15 @@
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
- TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+ TokenStream tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME, new StringReader(text)));
Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
- HighlighterTest.this);
+ HighlighterTest.this, false);
+
highlighter.setTextFragmenter(new SimpleFragmenter(40));
+ tokenStream.reset();
String highlightedText = highlighter.getBestFragments(tokenStream, text,
maxNumFragmentsRequired, "...");
+ tokenStream.reset();
System.out.println(highlightedText);
}
// We expect to have zero highlights if the query is multi-terms and is
@@ -1266,7 +1281,7 @@
// create an instance of the highlighter with the tags used to surround
// highlighted text
- Highlighter highlighter = new Highlighter(this, new QueryScorer(query));
+ Highlighter highlighter = new Highlighter(this, new QueryTermScorer(query));
for (int i = 0; i < hits.length(); i++) {
String text = hits.doc(i).get(FIELD_NAME);
@@ -1293,9 +1308,10 @@
if (mode == this.SPAN) {
TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
- fieldSpecificScorer = new SpanScorer(query, FIELD_NAME, ctf);
+ fieldSpecificScorer = new QueryScorer(query, FIELD_NAME);
+ ((QueryScorer) fieldSpecificScorer).setTokenStream(ctf);
} else if (mode == this.STANDARD) {
- fieldSpecificScorer = new QueryScorer(query, "contents");
+ fieldSpecificScorer = new QueryTermScorer(query, "contents");
}
Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
fieldSpecificScorer);
@@ -1308,9 +1324,10 @@
if (mode == this.SPAN) {
TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
- fieldInSpecificScorer = new SpanScorer(query, null, ctf);
+ fieldInSpecificScorer = new QueryScorer(query, null);
+ ((QueryScorer) fieldInSpecificScorer).setTokenStream(ctf);
} else if (mode == this.STANDARD) {
- fieldInSpecificScorer = new QueryScorer(query);
+ fieldInSpecificScorer = new QueryTermScorer(query);
}
Highlighter fieldInSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
@@ -1535,9 +1552,9 @@
Query query = parser.parse( q );
IndexSearcher searcher = new IndexSearcher( dir );
// This scorer can return negative idf -> null fragment
- Scorer scorer = new QueryScorer( query, searcher.getIndexReader(), "t_text1" );
+ Scorer scorer = new QueryTermScorer( query, searcher.getIndexReader(), "t_text1" );
// This scorer doesn't use idf (patch version)
- //Scorer scorer = new QueryScorer( query, "t_text1" );
+ //Scorer scorer = new QueryTermScorer( query, "t_text1" );
Highlighter h = new Highlighter( scorer );
TopDocs hits = searcher.search(query, null, 10);
@@ -1606,10 +1623,10 @@
String text = hits.doc(i).get(FIELD_NAME);
CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
new StringReader(text)));
- Highlighter highlighter = new Highlighter(this,
- new SpanScorer(query, FIELD_NAME, tokenStream));
+ QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+ Highlighter highlighter = new Highlighter(this, scorer);
+ scorer.setTokenStream(tokenStream);
highlighter.setTextFragmenter(new SimpleFragmenter(40));
- tokenStream.reset();
String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
"...");
@@ -1766,14 +1783,20 @@
public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
Formatter formatter) {
+ return getHighlighter(query, fieldName, stream, formatter, true);
+ }
+
+ public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
+ Formatter formatter, boolean expandMultiTerm) {
if (mode == STANDARD) {
- return new Highlighter(formatter, new QueryScorer(query));
+ return new Highlighter(formatter, new QueryTermScorer(query));
} else if (mode == SPAN) {
CachingTokenFilter tokenStream = new CachingTokenFilter(stream);
Highlighter highlighter;
+ QueryScorer scorer = new QueryScorer(query, fieldName, expandMultiTerm);
try {
- highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName, tokenStream));
- tokenStream.reset();
+ scorer.setTokenStream(tokenStream);
+ highlighter = new Highlighter(formatter, scorer);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -1786,11 +1809,11 @@
Highlighter getHighlighter(WeightedTerm[] weightedTerms, Formatter formatter) {
if (mode == STANDARD) {
- return new Highlighter(formatter, new QueryScorer(weightedTerms));
+ return new Highlighter(formatter, new QueryTermScorer(weightedTerms));
} else if (mode == SPAN) {
Highlighter highlighter;
- highlighter = new Highlighter(formatter, new SpanScorer((WeightedSpanTerm[]) weightedTerms));
+ highlighter = new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));
return highlighter;
} else {
@@ -1815,10 +1838,10 @@
if (mode == SPAN) {
tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
new StringReader(text)));
- scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME,
- (CachingTokenFilter) tokenStream, expandMT);
+ scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME, expandMT);
+ ((QueryScorer)scorer).setTokenStream((CachingTokenFilter) tokenStream);
} else if (mode == STANDARD) {
- scorer = new QueryScorer(query);
+ scorer = new QueryTermScorer(query);
tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
}
Highlighter highlighter = new Highlighter(formatter, scorer);