Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java	(working copy)
@@ -223,7 +223,10 @@
 	    tokenStream.reset();
 	    
 		TextFragment currentFrag =	new TextFragment(newText,newText.length(), docFrags.size());
-		fragmentScorer.init(tokenStream);
+		TokenStream newStream = fragmentScorer.init(tokenStream);
+		if(newStream != null) {
+		  tokenStream = newStream;
+		}
 		fragmentScorer.startFragment(currentFrag);
 		docFrags.add(currentFrag);
 
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java	(working copy)
@@ -1,161 +1,227 @@
 package org.apache.lucene.search.highlight;
 
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
+import java.io.IOException;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
 
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
 import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.search.Query;
 
 /**
  * {@link Scorer} implementation which scores text fragments by the number of
- * unique query terms found. This class uses the {@link QueryTermExtractor}
- * class to process determine the query terms and their boosts to be used.
+ * unique query terms found. This class converts appropriate Querys to
+ * SpanQuerys and attempts to score only those terms that participated in
+ * generating the 'hit' on the document.
  */
-// TODO: provide option to boost score of fragments near beginning of document
-// based on fragment.getFragNum()
 public class QueryScorer implements Scorer {
-  
-  TextFragment currentTextFragment = null;
-  HashSet uniqueTermsInFragment;
-
-  float totalScore = 0;
-  float maxTermWeight = 0;
-  private HashMap termsToFind;
-
+  private float totalScore;
+  private Set foundTerms;
+  private Map fieldWeightedSpanTerms;
+  private float maxTermWeight;
+  private int position = -1;
+  private String defaultField;
   private TermAttribute termAtt;
+  private PositionIncrementAttribute posIncAtt;
+  private boolean expandMultiTermQuery = true;
+  private Query query;
+  private String field;
+  private IndexReader reader;
+  private boolean skipInitExtractor;
 
   /**
+   * @param query Query to use for highlighting
    * 
-   * @param query a Lucene query (ideally rewritten using query.rewrite before
-   *        being passed to this class and the searcher)
+   * @throws IOException
    */
   public QueryScorer(Query query) {
-    this(QueryTermExtractor.getTerms(query));
+    init(query, null, null, true);
   }
 
   /**
-   * 
-   * @param query a Lucene query (ideally rewritten using query.rewrite before
-   *        being passed to this class and the searcher)
-   * @param fieldName the Field name which is used to match Query terms
+   * @param query Query to use for highlighting
+   * @param field Field to highlight - pass null to ignore fields
+   * @throws IOException
    */
-  public QueryScorer(Query query, String fieldName) {
-    this(QueryTermExtractor.getTerms(query, false, fieldName));
+  public QueryScorer(Query query, String field) {
+    init(query, field, null, true);
   }
 
   /**
+   * @param query Query to use for highlighting
+   * @param field Field to highlight - pass null to ignore fields
    * 
-   * @param query a Lucene query (ideally rewritten using query.rewrite before
-   *        being passed to this class and the searcher)
-   * @param reader used to compute IDF which can be used to a) score selected
-   *        fragments better b) use graded highlights eg set font color
-   *        intensity
-   * @param fieldName the field on which Inverse Document Frequency (IDF)
-   *        calculations are based
+   * @param reader
+   * @throws IOException
+   */
+  public QueryScorer(Query query, String field, IndexReader reader) {
+    init(query, field, reader, true);
+  }
+
+  /**
+   * As above, but with ability to pass in an <tt>IndexReader</tt>
    */
-  public QueryScorer(Query query, IndexReader reader, String fieldName) {
-    this(QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName));
+  public QueryScorer(Query query, String field, IndexReader reader, String defaultField)
+    throws IOException {
+    this.defaultField = defaultField.intern();
+    init(query, field, reader, true);
   }
 
-  public QueryScorer(WeightedTerm[] weightedTerms) {
-    termsToFind = new HashMap();
+  /**
+   * @param defaultField - The default field for queries with the field name unspecified
+   */
+  public QueryScorer(Query query, String field, String defaultField) {
+    this.defaultField = defaultField.intern();
+    init(query, field, null, true);
+  }
+
+  /**
+   * @param weightedTerms
+   */
+  public QueryScorer(WeightedSpanTerm[] weightedTerms) {
+    this.fieldWeightedSpanTerms = new HashMap(weightedTerms.length);
+
     for (int i = 0; i < weightedTerms.length; i++) {
-      WeightedTerm existingTerm = (WeightedTerm) termsToFind
-          .get(weightedTerms[i].term);
-      if ((existingTerm == null)
-          || (existingTerm.weight < weightedTerms[i].weight)) {
-        // if a term is defined more than once, always use the highest scoring
-        // weight
-        termsToFind.put(weightedTerms[i].term, weightedTerms[i]);
+      WeightedSpanTerm existingTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(weightedTerms[i].term);
+
+      if ((existingTerm == null) ||
+            (existingTerm.weight < weightedTerms[i].weight)) {
+        // if a term is defined more than once, always use the highest
+        // scoring weight
+        fieldWeightedSpanTerms.put(weightedTerms[i].term, weightedTerms[i]);
         maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
       }
     }
-  }
-
-  /* (non-Javadoc)
-   * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
-   */
-  public void init(TokenStream tokenStream) {
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+    skipInitExtractor = true;
   }
 
   /*
    * (non-Javadoc)
-   * 
-   * @see
-   * org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
-   * .lucene.search.highlight.TextFragment)
+   *
+   * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
    */
-  public void startFragment(TextFragment newFragment) {
-    uniqueTermsInFragment = new HashSet();
-    currentTextFragment = newFragment;
-    totalScore = 0;
-
+  public float getFragmentScore() {
+    return totalScore;
   }
 
+  /**
+   *
+   * @return The highest weighted term (useful for passing to
+   *         GradientFormatter to set top end of coloring scale.
+   */
+  public float getMaxTermWeight() {
+    return maxTermWeight;
+  }
 
-  /* (non-Javadoc)
-   * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
+  /*
+   * (non-Javadoc)
+   *
+   * @see org.apache.lucene.search.highlight.Scorer#getTokenScore(org.apache.lucene.analysis.Token,
+   *      int)
    */
   public float getTokenScore() {
+    position += posIncAtt.getPositionIncrement();
     String termText = termAtt.term();
 
-    WeightedTerm queryTerm = (WeightedTerm) termsToFind.get(termText);
-    if (queryTerm == null) {
-      // not a query term - return
+    WeightedSpanTerm weightedSpanTerm;
+
+    if ((weightedSpanTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(
+              termText)) == null) {
       return 0;
     }
+
+    if (weightedSpanTerm.positionSensitive &&
+          !weightedSpanTerm.checkPosition(position)) {
+      return 0;
+    }
+
+    float score = weightedSpanTerm.getWeight();
+
     // found a query term - is it unique in this doc?
-    if (!uniqueTermsInFragment.contains(termText)) {
-      totalScore += queryTerm.getWeight();
-      uniqueTermsInFragment.add(termText);
+    if (!foundTerms.contains(termText)) {
+      totalScore += score;
+      foundTerms.add(termText);
     }
-    return queryTerm.getWeight();
+
+    return score;
   }
 
+  public TokenStream init(TokenStream tokenStream) throws IOException {
+    position = -1;
+    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+    posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
+    if(!skipInitExtractor) {
+      if(fieldWeightedSpanTerms != null) {
+        fieldWeightedSpanTerms.clear();
+      }
+      return initExtractor(tokenStream);
+    }
+    return null;
+  }
+  
+  /**
+   * Retrieve the WeightedSpanTerm for the specified token. Useful for passing
+   * Span information to a Fragmenter.
+   *
+   * @param token
+   * @return WeightedSpanTerm for token
+   */
+  public WeightedSpanTerm getWeightedSpanTerm(String token) {
+    return (WeightedSpanTerm) fieldWeightedSpanTerms.get(token);
+  }
 
-  /* (non-Javadoc)
-   * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
+  /**
+   * @param query
+   * @param field
+   * @param tokenStream
+   * @param reader
+   * @throws IOException
    */
-  public float getFragmentScore() {
-    return totalScore;
+  private void init(Query query, String field, IndexReader reader, boolean expandMultiTermQuery) {
+    this.reader = reader;
+    this.expandMultiTermQuery = expandMultiTermQuery;
+    this.query = query;
+    this.field = field;
   }
+  
+  private TokenStream initExtractor(TokenStream tokenStream) throws IOException {
+    WeightedSpanTermExtractor qse = defaultField == null ? new WeightedSpanTermExtractor()
+        : new WeightedSpanTermExtractor(defaultField);
+
+    qse.setExpandMultiTermQuery(expandMultiTermQuery);
+    if (reader == null) {
+      this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query,
+          tokenStream, field);
+    } else {
+      this.fieldWeightedSpanTerms = qse.getWeightedSpanTermsWithScores(query,
+          tokenStream, field, reader);
+    }
+    if(qse.isCachedTokenStream()) {
+      return qse.getTokenStream();
+    }
+    
+    return null;
+  }
 
   /*
    * (non-Javadoc)
-   * 
-   * @see
-   * org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+   *
+   * @see org.apache.lucene.search.highlight.Scorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
    */
-  public void allFragmentsProcessed() {
-    // this class has no special operations to perform at end of processing
+  public void startFragment(TextFragment newFragment) {
+    foundTerms = new HashSet();
+    totalScore = 0;
+  }
+  
+  public boolean isExpandMultiTermQuery() {
+    return expandMultiTermQuery;
   }
 
-  /**
-   * 
-   * @return The highest weighted term (useful for passing to GradientFormatter
-   *         to set top end of coloring scale.
-   */
-  public float getMaxTermWeight() {
-    return maxTermWeight;
+  public void setExpandMultiTermQuery(boolean expandMultiTermQuery) {
+    this.expandMultiTermQuery = expandMultiTermQuery;
   }
 }
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java	(revision 0)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java	(revision 0)
@@ -0,0 +1,162 @@
+package org.apache.lucene.search.highlight;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Query;
+
+/**
+ * {@link Scorer} implementation which scores text fragments by the number of
+ * unique query terms found. This class uses the {@link QueryTermExtractor}
+ * class to process determine the query terms and their boosts to be used.
+ */
+// TODO: provide option to boost score of fragments near beginning of document
+// based on fragment.getFragNum()
+public class QueryTermScorer implements Scorer {
+  
+  TextFragment currentTextFragment = null;
+  HashSet uniqueTermsInFragment;
+
+  float totalScore = 0;
+  float maxTermWeight = 0;
+  private HashMap termsToFind;
+
+  private TermAttribute termAtt;
+
+  /**
+   * 
+   * @param query a Lucene query (ideally rewritten using query.rewrite before
+   *        being passed to this class and the searcher)
+   */
+  public QueryTermScorer(Query query) {
+    this(QueryTermExtractor.getTerms(query));
+  }
+
+  /**
+   * 
+   * @param query a Lucene query (ideally rewritten using query.rewrite before
+   *        being passed to this class and the searcher)
+   * @param fieldName the Field name which is used to match Query terms
+   */
+  public QueryTermScorer(Query query, String fieldName) {
+    this(QueryTermExtractor.getTerms(query, false, fieldName));
+  }
+
+  /**
+   * 
+   * @param query a Lucene query (ideally rewritten using query.rewrite before
+   *        being passed to this class and the searcher)
+   * @param reader used to compute IDF which can be used to a) score selected
+   *        fragments better b) use graded highlights eg set font color
+   *        intensity
+   * @param fieldName the field on which Inverse Document Frequency (IDF)
+   *        calculations are based
+   */
+  public QueryTermScorer(Query query, IndexReader reader, String fieldName) {
+    this(QueryTermExtractor.getIdfWeightedTerms(query, reader, fieldName));
+  }
+
+  public QueryTermScorer(WeightedTerm[] weightedTerms) {
+    termsToFind = new HashMap();
+    for (int i = 0; i < weightedTerms.length; i++) {
+      WeightedTerm existingTerm = (WeightedTerm) termsToFind
+          .get(weightedTerms[i].term);
+      if ((existingTerm == null)
+          || (existingTerm.weight < weightedTerms[i].weight)) {
+        // if a term is defined more than once, always use the highest scoring
+        // weight
+        termsToFind.put(weightedTerms[i].term, weightedTerms[i]);
+        maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
+      }
+    }
+  }
+
+  /* (non-Javadoc)
+   * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
+   */
+  public TokenStream init(TokenStream tokenStream) {
+    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
+    return null;
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see
+   * org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
+   * .lucene.search.highlight.TextFragment)
+   */
+  public void startFragment(TextFragment newFragment) {
+    uniqueTermsInFragment = new HashSet();
+    currentTextFragment = newFragment;
+    totalScore = 0;
+
+  }
+
+
+  /* (non-Javadoc)
+   * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
+   */
+  public float getTokenScore() {
+    String termText = termAtt.term();
+
+    WeightedTerm queryTerm = (WeightedTerm) termsToFind.get(termText);
+    if (queryTerm == null) {
+      // not a query term - return
+      return 0;
+    }
+    // found a query term - is it unique in this doc?
+    if (!uniqueTermsInFragment.contains(termText)) {
+      totalScore += queryTerm.getWeight();
+      uniqueTermsInFragment.add(termText);
+    }
+    return queryTerm.getWeight();
+  }
+
+
+  /* (non-Javadoc)
+   * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
+   */
+  public float getFragmentScore() {
+    return totalScore;
+  }
+
+  /*
+   * (non-Javadoc)
+   * 
+   * @see
+   * org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+   */
+  public void allFragmentsProcessed() {
+    // this class has no special operations to perform at end of processing
+  }
+
+  /**
+   * 
+   * @return The highest weighted term (useful for passing to GradientFormatter
+   *         to set top end of coloring scale.
+   */
+  public float getMaxTermWeight() {
+    return maxTermWeight;
+  }
+}
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/Scorer.java	(working copy)
@@ -17,6 +17,8 @@
  * limitations under the License.
  */
 
+import java.io.IOException;
+
 import org.apache.lucene.analysis.TokenStream;
 
 /**
@@ -30,8 +32,9 @@
    * getTokenScore().
    * 
    * @param tokenStream
+   * @throws IOException 
    */
-  public void init(TokenStream tokenStream);
+  public TokenStream init(TokenStream tokenStream) throws IOException;
 
   /**
    * called when a new fragment is started for consideration
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java	(working copy)
@@ -34,7 +34,7 @@
   private int fragmentSize;
   private int currentNumFrags;
   private int position = -1;
-  private SpanScorer spanScorer;
+  private QueryScorer QueryScorer;
   private int waitForPos = -1;
   private int textSize;
   private TermAttribute termAtt;
@@ -42,19 +42,19 @@
   private OffsetAttribute offsetAtt;
 
   /**
-   * @param spanscorer SpanScorer that was used to score hits
+   * @param QueryScorer QueryScorer that was used to score hits
    */
-  public SimpleSpanFragmenter(SpanScorer spanscorer) {
-    this(spanscorer, DEFAULT_FRAGMENT_SIZE);
+  public SimpleSpanFragmenter(QueryScorer QueryScorer) {
+    this(QueryScorer, DEFAULT_FRAGMENT_SIZE);
   }
 
   /**
-   * @param spanscorer SpanScorer that was used to score hits
+   * @param QueryScorer QueryScorer that was used to score hits
    * @param fragmentSize size in bytes of each fragment
    */
-  public SimpleSpanFragmenter(SpanScorer spanscorer, int fragmentSize) {
+  public SimpleSpanFragmenter(QueryScorer QueryScorer, int fragmentSize) {
     this.fragmentSize = fragmentSize;
-    this.spanScorer = spanscorer;
+    this.QueryScorer = QueryScorer;
   }
   
   /* (non-Javadoc)
@@ -69,7 +69,7 @@
       return false;
     }
 
-    WeightedSpanTerm wSpanTerm = spanScorer.getWeightedSpanTerm(termAtt.term());
+    WeightedSpanTerm wSpanTerm = QueryScorer.getWeightedSpanTerm(termAtt.term());
 
     if (wSpanTerm != null) {
       List positionSpans = wSpanTerm.getPositionSpans();
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java	(working copy)
@@ -1,288 +0,0 @@
-package org.apache.lucene.search.highlight;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.Query;
-
-
-/**
- * {@link Scorer} implementation which scores text fragments by the number of
- * unique query terms found. This class converts appropriate Querys to
- * SpanQuerys and attempts to score only those terms that participated in
- * generating the 'hit' on the document.
- */
-public class SpanScorer implements Scorer {
-  private float totalScore;
-  private Set foundTerms;
-  private Map fieldWeightedSpanTerms;
-  private float maxTermWeight;
-  private int position = -1;
-  private String defaultField;
-  private TermAttribute termAtt;
-  private PositionIncrementAttribute posIncAtt;
-  private static boolean highlightCnstScrRngQuery;
-
-  /**
-   * @param query
-   *            Query to use for highlighting
-   * @param field
-   *            Field to highlight - pass null to ignore fields
-   * @param tokenStream
-   *            of source text to be highlighted
-   * @throws IOException
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter) throws IOException {
-    init(query, field, cachingTokenFilter, null, false);
-  }
-  
-
-  /**
-   * @param query
-   *          Query to use for highlighting
-   * @param field
-   *          Field to highlight - pass null to ignore fields
-   * @param tokenStream
-   *          of source text to be highlighted
-   * @param expandMultiTermQuery
-   *          rewrite multi-term queries against a single doc memory index to
-   *          create boolean queries
-   * @throws IOException
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, boolean expandMultiTermQuery) throws IOException {
-    init(query, field, cachingTokenFilter, null, expandMultiTermQuery);
-  }
-
-  /**
-   * @param query
-   *            Query to use for highlighting
-   * @param field
-   *            Field to highlight - pass null to ignore fields
-   * @param tokenStream
-   *            of source text to be highlighted
-   * @param reader
-   * @throws IOException
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, IndexReader reader)
-    throws IOException {
-    init(query, field, cachingTokenFilter, reader, false);
-  }
-  
-  /**
-   * @param query
-   *            Query to use for highlighting
-   * @param field
-   *            Field to highlight - pass null to ignore fields
-   * @param tokenStream
-   *            of source text to be highlighted
-   * @param reader
-   * @param expandMultiTermQuery
-   *          rewrite multi-term queries against a single doc memory index to
-   *          create boolean queries
-   * @throws IOException
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, IndexReader reader, boolean expandMultiTermQuery)
-    throws IOException {
-    init(query, field, cachingTokenFilter, reader, expandMultiTermQuery);
-  }
-
-  /**
-   * As above, but with ability to pass in an <tt>IndexReader</tt>
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, IndexReader reader, String defaultField)
-    throws IOException {
-    this.defaultField = defaultField.intern();
-    init(query, field, cachingTokenFilter, reader, false);
-  }
-  
-  /**
-   * As above, but with ability to pass in an <tt>IndexReader</tt>
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, IndexReader reader, String defaultField, boolean expandMultiTermQuery)
-    throws IOException {
-    this.defaultField = defaultField.intern();
-    init(query, field, cachingTokenFilter, reader, expandMultiTermQuery);
-  }
-
-  /**
-   * @param defaultField - The default field for queries with the field name unspecified
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, String defaultField) throws IOException {
-    this.defaultField = defaultField.intern();
-    init(query, field, cachingTokenFilter, null, false);
-  }
-  
-  /**
-   * @param defaultField - The default field for queries with the field name unspecified
-   */
-  public SpanScorer(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, String defaultField, boolean expandMultiTermQuery) throws IOException {
-    this.defaultField = defaultField.intern();
-    init(query, field, cachingTokenFilter, null, expandMultiTermQuery);
-  }
-
-  /**
-   * @param weightedTerms
-   */
-  public SpanScorer(WeightedSpanTerm[] weightedTerms) {
-    this.fieldWeightedSpanTerms = new HashMap(weightedTerms.length);
-
-    for (int i = 0; i < weightedTerms.length; i++) {
-      WeightedSpanTerm existingTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(weightedTerms[i].term);
-
-      if ((existingTerm == null) ||
-            (existingTerm.weight < weightedTerms[i].weight)) {
-        // if a term is defined more than once, always use the highest
-        // scoring weight
-        fieldWeightedSpanTerms.put(weightedTerms[i].term, weightedTerms[i]);
-        maxTermWeight = Math.max(maxTermWeight, weightedTerms[i].getWeight());
-      }
-    }
-  }
-
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
-   */
-  public float getFragmentScore() {
-    return totalScore;
-  }
-
-  /**
-   *
-   * @return The highest weighted term (useful for passing to
-   *         GradientFormatter to set top end of coloring scale.
-   */
-  public float getMaxTermWeight() {
-    return maxTermWeight;
-  }
-
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lucene.search.highlight.Scorer#getTokenScore(org.apache.lucene.analysis.Token,
-   *      int)
-   */
-  public float getTokenScore() {
-    position += posIncAtt.getPositionIncrement();
-    String termText = termAtt.term();
-
-    WeightedSpanTerm weightedSpanTerm;
-
-    if ((weightedSpanTerm = (WeightedSpanTerm) fieldWeightedSpanTerms.get(
-              termText)) == null) {
-      return 0;
-    }
-
-    if (weightedSpanTerm.positionSensitive &&
-          !weightedSpanTerm.checkPosition(position)) {
-      return 0;
-    }
-
-    float score = weightedSpanTerm.getWeight();
-
-    // found a query term - is it unique in this doc?
-    if (!foundTerms.contains(termText)) {
-      totalScore += score;
-      foundTerms.add(termText);
-    }
-
-    return score;
-  }
-
-  public void init(TokenStream tokenStream) {
-    termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-    posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class);
-  }
-  
-  /**
-   * Retrieve the WeightedSpanTerm for the specified token. Useful for passing
-   * Span information to a Fragmenter.
-   *
-   * @param token
-   * @return WeightedSpanTerm for token
-   */
-  public WeightedSpanTerm getWeightedSpanTerm(String token) {
-    return (WeightedSpanTerm) fieldWeightedSpanTerms.get(token);
-  }
-
-  /**
-   * @param query
-   * @param field
-   * @param tokenStream
-   * @param reader
-   * @throws IOException
-   */
-  private void init(Query query, String field,
-    CachingTokenFilter cachingTokenFilter, IndexReader reader, boolean expandMultiTermQuery)
-    throws IOException {
-    WeightedSpanTermExtractor qse = defaultField == null ? new WeightedSpanTermExtractor()
-      : new WeightedSpanTermExtractor(defaultField);
-    
-    qse.setHighlightCnstScrRngQuery(highlightCnstScrRngQuery);
-    qse.setExpandMultiTermQuery(expandMultiTermQuery);
-    if (reader == null) {
-      this.fieldWeightedSpanTerms = qse.getWeightedSpanTerms(query,
-          cachingTokenFilter, field);
-    } else {
-      this.fieldWeightedSpanTerms = qse.getWeightedSpanTermsWithScores(query,
-          cachingTokenFilter, field, reader);
-    }
-  }
-
-  /**
-   * @return whether ConstantScoreRangeQuerys are set to be highlighted
-   * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
-   *             constructor option to expand MultiTerm queries.
-   */
-  public static boolean isHighlightCnstScrRngQuery() {
-    return highlightCnstScrRngQuery;
-  }
-
-  /**
-   * If you call Highlighter#getBestFragment() more than once you must reset
-   * the SpanScorer between each call.
-   */
-  public void reset() {
-    position = -1;
-  }
-
-  /**
-   * Turns highlighting of ConstantScoreRangeQuery on/off.
-   * ConstantScoreRangeQuerys cannot be highlighted if you rewrite the query
-   * first. Must be called before SpanScorer construction.
-   * 
-   * @param highlightCnstScrRngQuery
-   * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
-   *             constructor option to expand MultiTerm queries.
-   */
-  public static void setHighlightCnstScrRngQuery(boolean highlight) {
-    highlightCnstScrRngQuery = highlight;
-  }
-
-  /*
-   * (non-Javadoc)
-   *
-   * @see org.apache.lucene.search.highlight.Scorer#startFragment(org.apache.lucene.search.highlight.TextFragment)
-   */
-  public void startFragment(TextFragment newFragment) {
-    foundTerms = new HashSet();
-    totalScore = 0;
-  }
-}
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java	(revision 799886)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java	(working copy)
@@ -27,6 +27,7 @@
 import java.util.Set;
 
 import org.apache.lucene.analysis.CachingTokenFilter;
+import org.apache.lucene.analysis.TokenFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
@@ -57,11 +58,11 @@
 public class WeightedSpanTermExtractor {
 
   private String fieldName;
-  private CachingTokenFilter cachedTokenFilter;
+  private TokenStream tokenStream;
   private Map readers = new HashMap(10); // Map<String, IndexReader>
   private String defaultField;
-  private boolean highlightCnstScrRngQuery;
   private boolean expandMultiTermQuery;
+  private boolean cachedTokenStream;
 
   public WeightedSpanTermExtractor() {
   }
@@ -131,7 +132,7 @@
       for (Iterator iterator = ((DisjunctionMaxQuery) query).iterator(); iterator.hasNext();) {
         extract((Query) iterator.next(), terms);
       }
-    } else if (query instanceof MultiTermQuery && (highlightCnstScrRngQuery || expandMultiTermQuery)) {
+    } else if (query instanceof MultiTermQuery && expandMultiTermQuery) {
       MultiTermQuery mtq = ((MultiTermQuery)query);
       if(mtq.getRewriteMethod() != MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE) {
         mtq = copyMultiTermQuery(mtq);
@@ -240,8 +241,7 @@
       while (spans.next()) {
         spanPositions.add(new PositionSpan(spans.start(), spans.end() - 1));
       }
-
-      cachedTokenFilter.reset();
+      
     }
 
     if (spanPositions.size() == 0) {
@@ -301,15 +301,21 @@
     return rv;
   }
 
-  private IndexReader getReaderForField(String field) {
+  private IndexReader getReaderForField(String field) throws IOException {
+    if(!cachedTokenStream && !(tokenStream instanceof CachingTokenFilter)) {
+      tokenStream = new CachingTokenFilter(tokenStream);
+      cachedTokenStream = true;
+    }
     IndexReader reader = (IndexReader) readers.get(field);
     if (reader == null) {
       MemoryIndex indexer = new MemoryIndex();
-      indexer.addField(field, cachedTokenFilter);
+      indexer.addField(field, tokenStream);
+      tokenStream.reset();
       IndexSearcher searcher = indexer.createSearcher();
       reader = searcher.getIndexReader();
       readers.put(field, reader);
     }
+
     return reader;
   }
 
@@ -328,7 +334,7 @@
   public Map getWeightedSpanTerms(Query query, CachingTokenFilter cachingTokenFilter)
       throws IOException {
     this.fieldName = null;
-    this.cachedTokenFilter = cachingTokenFilter;
+    this.tokenStream = cachingTokenFilter;
 
     Map terms = new PositionCheckingMap();
     try {
@@ -354,14 +360,14 @@
    * @return
    * @throws IOException
    */
-  public Map getWeightedSpanTerms(Query query, CachingTokenFilter cachingTokenFilter,
+  public Map getWeightedSpanTerms(Query query, TokenStream tokenStream,
       String fieldName) throws IOException {
     if (fieldName != null) {
       this.fieldName = fieldName.intern();
     }
 
     Map terms = new PositionCheckingMap();
-    this.cachedTokenFilter = cachingTokenFilter;
+    this.tokenStream = tokenStream;
     try {
       extract(query, terms);
     } finally {
@@ -391,7 +397,7 @@
   public Map getWeightedSpanTermsWithScores(Query query, TokenStream tokenStream, String fieldName,
       IndexReader reader) throws IOException {
     this.fieldName = fieldName;
-    this.cachedTokenFilter = new CachingTokenFilter(tokenStream);
+    this.tokenStream = tokenStream;
 
     Map terms = new PositionCheckingMap();
     extract(query, terms);
@@ -419,23 +425,6 @@
 
     return terms;
   }
-
-  /**
-   * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use
-   *             getExpandMultiTermQuery instead.
-   */
-  public boolean isHighlightCnstScrRngQuery() {
-    return highlightCnstScrRngQuery;
-  }
-  
-  /**
-   * @param highlightCnstScrRngQuery
-   * @deprecated {@link ConstantScoreRangeQuery} is deprecated. Use the
-   *             setExpandMultiTermQuery option.
-   */
-  public void setHighlightCnstScrRngQuery(boolean highlightCnstScrRngQuery) {
-    this.highlightCnstScrRngQuery = highlightCnstScrRngQuery;
-  }
   
   /**
    * This class makes sure that if both position sensitive and insensitive
@@ -495,4 +484,12 @@
   public void setExpandMultiTermQuery(boolean expandMultiTermQuery) {
     this.expandMultiTermQuery = expandMultiTermQuery;
   }
+  
+  public boolean isCachedTokenStream() {
+    return cachedTokenStream;
+  }
+  
+  public TokenStream getTokenStream() {
+    return tokenStream;
+  }
 }
Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
===================================================================
--- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(revision 799886)
+++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(working copy)
@@ -118,8 +118,9 @@
     query = qp.parse("\"very long\"");
     searcher = new IndexSearcher(ramDir, false);
     TopDocs hits = searcher.search(query, 10);
-
-    Highlighter highlighter = new Highlighter(null);
+    
+    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(scorer);
 
 
     for (int i = 0; i < hits.scoreDocs.length; i++) {
@@ -128,14 +129,12 @@
 
       TokenStream stream = TokenSources.getAnyTokenStream(searcher
           .getIndexReader(), hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
-      CachingTokenFilter ctf = new CachingTokenFilter(stream);
-      SpanScorer scorer = new SpanScorer(query, FIELD_NAME, ctf);
-     // ctf.reset();
+
       Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
-      highlighter.setFragmentScorer(scorer);
+
       highlighter.setTextFragmenter(fragmenter);
 
-      String fragment = highlighter.getBestFragment(ctf, storedField);
+      String fragment = highlighter.getBestFragment(stream, storedField);
 
       System.out.println(fragment);
     }
@@ -181,10 +180,10 @@
         fieldName, new StringReader(text)));
     // Assuming "<B>", "</B>" used to highlight
     SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
-    Highlighter highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName,
-        tokenStream, FIELD_NAME));
+    QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(formatter, scorer);
     highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
-    tokenStream.reset();
+
     String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");
     return rv.length() == 0 ? text : rv;
   }
@@ -194,13 +193,14 @@
 
     int maxNumFragmentsRequired = 2;
 
+    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(scorer);
+    
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
-      Highlighter highlighter = new Highlighter(new SpanScorer(query, FIELD_NAME, tokenStream));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+          new StringReader(text));
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -225,9 +225,10 @@
     Analyzer analyzer = new WhitespaceAnalyzer();
     QueryParser qp = new QueryParser(f1, analyzer);
     Query query = qp.parse(q);
-    CachingTokenFilter stream = new CachingTokenFilter(analyzer.tokenStream(f1,
-        new StringReader(content)));
-    Scorer scorer = new SpanScorer(query, f1, stream, false);
+
+    QueryScorer scorer = new QueryScorer(query, f1);
+    scorer.setExpandMultiTermQuery(false);
+
     Highlighter h = new Highlighter(this, scorer);
 
     h.getBestFragment(analyzer, f1, content);
@@ -241,14 +242,14 @@
 
     int maxNumFragmentsRequired = 2;
 
+    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(this, scorer);
+    
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
-      Highlighter highlighter = new Highlighter(this,
-          new SpanScorer(query, FIELD_NAME, tokenStream));
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -264,14 +265,13 @@
 
     int maxNumFragmentsRequired = 2;
 
+    QueryScorer scorer =  new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(this,scorer);
+    highlighter.setTextFragmenter(new SimpleFragmenter(40));
+    
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
-      CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text)));
-      Highlighter highlighter = new Highlighter(this,
-          new SpanScorer(query, FIELD_NAME, tokenStream));
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
+      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -291,10 +291,10 @@
       String text = hits.doc(i).get(FIELD_NAME);
       CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
           new StringReader(text)));
-      Highlighter highlighter = new Highlighter(this,
-          new SpanScorer(query, FIELD_NAME, tokenStream));
+      QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+      Highlighter highlighter = new Highlighter(this, scorer);
+
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -310,14 +310,15 @@
 
     int maxNumFragmentsRequired = 2;
 
+    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(this, scorer);
+    
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
       CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
           .tokenStream(FIELD_NAME, new StringReader(text)));
-      SpanScorer spanscorer = new SpanScorer(query, FIELD_NAME, tokenStream);
-      Highlighter highlighter = new Highlighter(this, spanscorer);
-      highlighter.setTextFragmenter(new SimpleSpanFragmenter(spanscorer, 5));
-      tokenStream.reset();
+
+      highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
 
       String result = highlighter.getBestFragments(tokenStream, text,
           maxNumFragmentsRequired, "...");
@@ -328,15 +329,16 @@
     doSearching("\"been shot\"");
 
     maxNumFragmentsRequired = 2;
+    
+    scorer = new QueryScorer(query, FIELD_NAME);
+    highlighter = new Highlighter(this, scorer);
 
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
       CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer
           .tokenStream(FIELD_NAME, new StringReader(text)));
-      SpanScorer spanscorer = new SpanScorer(query, FIELD_NAME, tokenStream);
-      Highlighter highlighter = new Highlighter(this, spanscorer);
-      highlighter.setTextFragmenter(new SimpleSpanFragmenter(spanscorer, 20));
-      tokenStream.reset();
+
+      highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
 
       String result = highlighter.getBestFragments(tokenStream, text,
           maxNumFragmentsRequired, "...");
@@ -350,15 +352,16 @@
     doSearching("y \"x y z\"");
 
     int maxNumFragmentsRequired = 2;
-
+    
+    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+    Highlighter highlighter = new Highlighter(this,scorer);
+    
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
       CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
           new StringReader(text)));
-      Highlighter highlighter = new Highlighter(this,
-          new SpanScorer(query, FIELD_NAME, tokenStream));
+
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -421,7 +424,7 @@
 
   public void testSimpleHighlighter() throws Exception {
     doSearching("Kennedy");
-    Highlighter highlighter = new Highlighter(new QueryScorer(query));
+    Highlighter highlighter = new Highlighter(new QueryTermScorer(query));
     highlighter.setTextFragmenter(new SimpleFragmenter(40));
     int maxNumFragmentsRequired = 2;
     for (int i = 0; i < hits.length(); i++) {
@@ -579,18 +582,15 @@
       String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
       int maxNumFragmentsRequired = 2;
       String fragmentSeparator = "...";
-      SpanScorer scorer = null;
+      QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
       tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
           new StringReader(text)));
       
-      SpanScorer.setHighlightCnstScrRngQuery(true);
-      scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME, (CachingTokenFilter) tokenStream);
-      
-      Highlighter highlighter = new Highlighter(this, scorer);
+      scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
-      ((CachingTokenFilter) tokenStream).reset();
+      Highlighter highlighter = new Highlighter(this, scorer);
 
       highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
@@ -619,18 +619,16 @@
       String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
       int maxNumFragmentsRequired = 2;
       String fragmentSeparator = "...";
-      SpanScorer scorer = null;
+      QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
       tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
           new StringReader(text)));
       
-      scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME, (CachingTokenFilter) tokenStream, true);
+      scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
       Highlighter highlighter = new Highlighter(this, scorer);
 
-      ((CachingTokenFilter) tokenStream).reset();
-
       highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -650,18 +648,16 @@
       String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
       int maxNumFragmentsRequired = 2;
       String fragmentSeparator = "...";
-      SpanScorer scorer = null;
+      QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
       tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
           new StringReader(text)));
       
-      scorer = new SpanScorer(query, null, (CachingTokenFilter) tokenStream, true);
+      scorer = new QueryScorer(query, null);
 
       Highlighter highlighter = new Highlighter(this, scorer);
 
-      ((CachingTokenFilter) tokenStream).reset();
-
       highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -681,18 +677,16 @@
       String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
       int maxNumFragmentsRequired = 2;
       String fragmentSeparator = "...";
-      SpanScorer scorer = null;
+      QueryScorer scorer = null;
       TokenStream tokenStream = null;
 
       tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
           new StringReader(text)));
       
-      scorer = new SpanScorer(query, "random_field", (CachingTokenFilter) tokenStream, HighlighterTest.FIELD_NAME, true);
+      scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
 
       Highlighter highlighter = new Highlighter(this, scorer);
 
-      ((CachingTokenFilter) tokenStream).reset();
-
       highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
@@ -744,7 +738,7 @@
 
       public void run() throws Exception {
         TermQuery query = new TermQuery(new Term("data", "help"));
-        Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer(query));
+        Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(query));
         hg.setTextFragmenter(new NullFragmenter());
 
         String match = null;
@@ -900,7 +894,7 @@
 
         Highlighter highlighter = getHighlighter(wTerms, HighlighterTest.this);// new
         // Highlighter(new
-        // QueryScorer(wTerms));
+        // QueryTermScorer(wTerms));
         TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
         highlighter.setTextFragmenter(new SimpleFragmenter(2));
 
@@ -965,7 +959,7 @@
       public void run() throws Exception {
         numHighlights = 0;
         doSearching("Kennedy");
-        // new Highlighter(HighlighterTest.this, new QueryScorer(query));
+        // new Highlighter(HighlighterTest.this, new QueryTermScorer(query));
 
         for (int i = 0; i < hits.length(); i++) {
           String text = hits.doc(i).get(FIELD_NAME);
@@ -995,7 +989,7 @@
 
           Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
               HighlighterTest.this);// new Highlighter(this, new
-          // QueryScorer(query));
+          // QueryTermScorer(query));
           highlighter.setTextFragmenter(new SimpleFragmenter(20));
           String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);
 
@@ -1027,7 +1021,7 @@
         TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
         Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
             HighlighterTest.this);// new Highlighter(this, new
-        // QueryScorer(query));
+        // QueryTermScorer(query));
         highlighter.setMaxDocBytesToAnalyze(30);
 
         highlighter.getBestFragment(tokenStream, texts[0]);
@@ -1062,7 +1056,7 @@
         Highlighter hg = getHighlighter(query, "data", new StandardAnalyzer(stopWords).tokenStream(
             "data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
         // new
-        // QueryScorer(query));
+        // QueryTermScorer(query));
         hg.setTextFragmenter(new NullFragmenter());
         hg.setMaxDocBytesToAnalyze(100);
         match = hg.getBestFragment(new StandardAnalyzer(stopWords), "data", sb.toString());
@@ -1114,7 +1108,6 @@
 
       public void run() throws Exception {
         numHighlights = 0;
-        SpanScorer.setHighlightCnstScrRngQuery(false);
         // test to show how rewritten query can still be used
         searcher = new IndexSearcher(ramDir);
         Analyzer analyzer = new StandardAnalyzer();
@@ -1136,12 +1129,14 @@
 
         for (int i = 0; i < hits.length(); i++) {
           String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
+          TokenStream tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME, new StringReader(text)));
+          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);
+
           highlighter.setTextFragmenter(new SimpleFragmenter(40));
+
           String highlightedText = highlighter.getBestFragments(tokenStream, text,
               maxNumFragmentsRequired, "...");
+
           System.out.println(highlightedText);
         }
         // We expect to have zero highlights if the query is multi-terms and is
@@ -1198,8 +1193,8 @@
         return 1;
       }
 
-      public void init(TokenStream tokenStream) {
-        
+      public TokenStream init(TokenStream tokenStream) {
+        return null;
       }
     });
     highlighter.setTextFragmenter(new SimpleFragmenter(2000));
@@ -1266,7 +1261,7 @@
 
     // create an instance of the highlighter with the tags used to surround
     // highlighted text
-    Highlighter highlighter = new Highlighter(this, new QueryScorer(query));
+    Highlighter highlighter = new Highlighter(this, new QueryTermScorer(query));
 
     for (int i = 0; i < hits.length(); i++) {
       String text = hits.doc(i).get(FIELD_NAME);
@@ -1293,9 +1288,10 @@
         if (mode == this.SPAN) {
           TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
           CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
-          fieldSpecificScorer = new SpanScorer(query, FIELD_NAME, ctf);
+          fieldSpecificScorer = new QueryScorer(query, FIELD_NAME);
+
         } else if (mode == this.STANDARD) {
-          fieldSpecificScorer = new QueryScorer(query, "contents");
+          fieldSpecificScorer = new QueryTermScorer(query, "contents");
         }
         Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
             fieldSpecificScorer);
@@ -1308,9 +1304,10 @@
         if (mode == this.SPAN) {
           TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(docMainText));
           CachingTokenFilter ctf = new CachingTokenFilter(tokenStream);
-          fieldInSpecificScorer = new SpanScorer(query, null, ctf);
+          fieldInSpecificScorer = new QueryScorer(query, null);
+
         } else if (mode == this.STANDARD) {
-          fieldInSpecificScorer = new QueryScorer(query);
+          fieldInSpecificScorer = new QueryTermScorer(query);
         }
 
         Highlighter fieldInSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
@@ -1535,9 +1532,9 @@
     Query query = parser.parse( q );
     IndexSearcher searcher = new IndexSearcher( dir );
     // This scorer can return negative idf -> null fragment
-    Scorer scorer = new QueryScorer( query, searcher.getIndexReader(), "t_text1" );
+    Scorer scorer = new QueryTermScorer( query, searcher.getIndexReader(), "t_text1" );
     // This scorer doesn't use idf (patch version)
-    //Scorer scorer = new QueryScorer( query, "t_text1" );
+    //Scorer scorer = new QueryTermScorer( query, "t_text1" );
     Highlighter h = new Highlighter( scorer );
 
     TopDocs hits = searcher.search(query, null, 10);
@@ -1606,10 +1603,10 @@
       String text = hits.doc(i).get(FIELD_NAME);
       CachingTokenFilter tokenStream = new CachingTokenFilter(analyzer.tokenStream(FIELD_NAME,
           new StringReader(text)));
-      Highlighter highlighter = new Highlighter(this,
-          new SpanScorer(query, FIELD_NAME, tokenStream));
+      QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+      Highlighter highlighter = new Highlighter(this, scorer);
+
       highlighter.setTextFragmenter(new SimpleFragmenter(40));
-      tokenStream.reset();
 
       String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
           "...");
@@ -1763,34 +1760,34 @@
     static final int SPAN = 1;
     int mode = STANDARD;
     Fragmenter frag = new SimpleFragmenter(20);
-
-    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream,
-        Formatter formatter) {
+    
+    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream, Formatter formatter) {
+      return getHighlighter(query, fieldName, stream, formatter, true);
+    }
+    
+    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream, Formatter formatter, boolean expanMultiTerm) {
+      Scorer scorer = null;
       if (mode == STANDARD) {
-        return new Highlighter(formatter, new QueryScorer(query));
+        scorer = new QueryTermScorer(query);
       } else if (mode == SPAN) {
-        CachingTokenFilter tokenStream = new CachingTokenFilter(stream);
-        Highlighter highlighter;
-        try {
-          highlighter = new Highlighter(formatter, new SpanScorer(query, fieldName, tokenStream));
-          tokenStream.reset();
-        } catch (IOException e) {
-          throw new RuntimeException(e);
+        scorer = new QueryScorer(query, fieldName);
+        if(!expanMultiTerm) {
+          ((QueryScorer)scorer).setExpandMultiTermQuery(false);
         }
-
-        return highlighter;
       } else {
         throw new RuntimeException("Unknown highlight mode");
       }
+      
+      return new Highlighter(formatter, scorer);
     }
 
     Highlighter getHighlighter(WeightedTerm[] weightedTerms, Formatter formatter) {
       if (mode == STANDARD) {
-        return new Highlighter(formatter, new QueryScorer(weightedTerms));
+        return new Highlighter(formatter, new QueryTermScorer(weightedTerms));
       } else if (mode == SPAN) {
         Highlighter highlighter;
 
-        highlighter = new Highlighter(formatter, new SpanScorer((WeightedSpanTerm[]) weightedTerms));
+        highlighter = new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));
 
         return highlighter;
       } else {
@@ -1815,16 +1812,14 @@
         if (mode == SPAN) {
           tokenStream = new CachingTokenFilter(analyzer.tokenStream(HighlighterTest.FIELD_NAME,
               new StringReader(text)));
-          scorer = new SpanScorer(query, HighlighterTest.FIELD_NAME,
-              (CachingTokenFilter) tokenStream, expandMT);
+          scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
+
         } else if (mode == STANDARD) {
-          scorer = new QueryScorer(query);
+          scorer = new QueryTermScorer(query);
           tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
         }
         Highlighter highlighter = new Highlighter(formatter, scorer);
-        if (mode == SPAN) {
-          ((CachingTokenFilter) tokenStream).reset();
-        }
+
         highlighter.setTextFragmenter(frag);
 
         String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,