Property changes on: src/test/org/apache/lucene/search/spans
___________________________________________________________________
Added: svn:ignore
   + .TestSpans.java.swp


Index: src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
===================================================================
--- src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java	(revision 908726)
+++ src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java	(working copy)
@@ -37,6 +37,10 @@
 
   static final class JustCompileSpans extends Spans {
 
+    public Spans[] getSubSpans() {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
     public int doc() {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
@@ -90,6 +94,10 @@
 
   static final class JustCompilePayloadSpans extends Spans {
 
+    public Spans[] getSubSpans() {
+      throw new UnsupportedOperationException(UNSUPPORTED_MSG);
+    }
+
     public Collection getPayload() throws IOException {
       throw new UnsupportedOperationException(UNSUPPORTED_MSG);
     }
Index: src/java/org/apache/lucene/search/spans/SpanFirstQuery.java
===================================================================
--- src/java/org/apache/lucene/search/spans/SpanFirstQuery.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/SpanFirstQuery.java	(working copy)
@@ -77,6 +77,10 @@
   public Spans getSpans(final IndexReader reader) throws IOException {
     return new Spans() {
         private Spans spans = match.getSpans(reader);
+        
+        public Spans[] getSubSpans() {
+        	return spans.getSubSpans();
+        }
 
         public boolean next() throws IOException {
           while (spans.next()) {                  // scan to next match
Index: src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
===================================================================
--- src/java/org/apache/lucene/search/spans/NearSpansUnordered.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/NearSpansUnordered.java	(working copy)
@@ -80,6 +80,10 @@
       this.index = index;
     }
 
+    public Spans[] getSubSpans() {
+    	return spans.getSubSpans();
+    }
+    
     public boolean next() throws IOException {
       return adjust(spans.next());
     }
Index: src/java/org/apache/lucene/search/spans/Spans.java
===================================================================
--- src/java/org/apache/lucene/search/spans/Spans.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/Spans.java	(working copy)
@@ -86,5 +86,7 @@
    * @return true if there is a payload available at this position that can be loaded
    */
   public abstract boolean isPayloadAvailable();
+  
+  public abstract Spans[] getSubSpans();
 
 }
Index: src/java/org/apache/lucene/search/spans/SpanNotQuery.java
===================================================================
--- src/java/org/apache/lucene/search/spans/SpanNotQuery.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/SpanNotQuery.java	(working copy)
@@ -81,6 +81,10 @@
 
         private Spans excludeSpans = exclude.getSpans(reader);
         private boolean moreExclude = excludeSpans.next();
+        
+        public Spans[] getSubSpans() {
+        	return includeSpans.getSubSpans();       	
+        }
 
         public boolean next() throws IOException {
           if (moreInclude)                        // move to next include
Index: src/java/org/apache/lucene/search/spans/SpanOrQuery.java
===================================================================
--- src/java/org/apache/lucene/search/spans/SpanOrQuery.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/SpanOrQuery.java	(working copy)
@@ -173,12 +173,20 @@
 
     return new Spans() {
         private SpanQueue queue = null;
+        private Spans[] subSpans = null;
+        
+        // MIKE: Adding this for highlighting
+        public Spans[] getSubSpans() {
+        	return subSpans;
+        }
 
         private boolean initSpanQueue(int target) throws IOException {
           queue = new SpanQueue(clauses.size());
+          subSpans = new Spans[clauses.size()];
           Iterator i = clauses.iterator();
-          while (i.hasNext()) {
+          for (int j = 0; i.hasNext(); j++) {
             Spans spans = ((SpanQuery)i.next()).getSpans(reader);
+            subSpans[j] = spans;
             if (   ((target == -1) && spans.next())
                 || ((target != -1) && spans.skipTo(target))) {
               queue.put(spans);
Index: src/java/org/apache/lucene/search/spans/TermSpans.java
===================================================================
--- src/java/org/apache/lucene/search/spans/TermSpans.java	(revision 908726)
+++ src/java/org/apache/lucene/search/spans/TermSpans.java	(working copy)
@@ -42,6 +42,14 @@
     this.term = term;
     doc = -1;
   }
+  
+  public Term getTerm() {
+	  return term;
+  }
+  
+  public Spans[] getSubSpans() {
+	  return null;
+  }
 
   public boolean next() throws IOException {
     if (count == freq) {
Index: build.xml
===================================================================
--- build.xml	(revision 908726)
+++ build.xml	(working copy)
@@ -666,6 +666,7 @@
   </target>
 
   <target name="test-contrib" depends="build-contrib">
+    <record name="test-contrib.log" action="start"/>
     <!-- Don't fail on error, instead check for flag file so we run
          all the tests possible and can "ant generate-test-reports"
          for all of them.
@@ -684,6 +685,7 @@
       </filepath>
     </available>
     <fail if="contribs.failed">Contrib tests failed!</fail>
+    <record name="test-contrib.log" action="stop"/>
   </target>
 
   <!-- Macro for building checksum files
Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
===================================================================
--- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(revision 908726)
+++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(working copy)
@@ -56,6 +56,7 @@
 import org.apache.lucene.queryParser.QueryParser;
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreRangeQuery;
+import org.apache.lucene.search.Explanation;
 import org.apache.lucene.search.FilteredQuery;
 import org.apache.lucene.search.Hits;
 import org.apache.lucene.search.IndexSearcher;
@@ -87,1661 +88,1927 @@
 
 /**
  * JUnit Test for Highlighter class.
- *
+ * 
  */
-public class HighlighterTest extends BaseTokenStreamTestCase implements Formatter {
-  private IndexReader reader;
-  static final String FIELD_NAME = "contents";
-  private static final String NUMERIC_FIELD_NAME = "nfield";
-  private Query query;
-  RAMDirectory ramDir;
-  public IndexSearcher searcher = null;
-  public Hits hits = null;
-  int numHighlights = 0;
-  Analyzer analyzer = new StandardAnalyzer();
+public class HighlighterTest extends BaseTokenStreamTestCase implements
+		Formatter {
+	private IndexReader reader;
+	static final String FIELD_NAME = "contents";
+	private static final String NUMERIC_FIELD_NAME = "nfield";
+	private Query query;
+	RAMDirectory ramDir;
+	public IndexSearcher searcher = null;
+	public Hits hits = null;
+	int numHighlights = 0;
+	Analyzer analyzer = new StandardAnalyzer();
 
-  String[] texts = {
-      "Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
-      "This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
-      "JFK has been shot", "John Kennedy has been shot",
-      "This text has a typo in referring to Keneddy",
-      "wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b" };
+	String[] texts = {
+			"Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
+			"This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
+			"JFK has been shot", "John Kennedy has been shot",
+			"This text has a typo in referring to Keneddy",
+			"wordx wordy wordz wordx wordy wordx worda wordb wordy wordc",
+			"y z x y z a b" };
 
-  /**
-   * Constructor for HighlightExtractorTest.
-   * 
-   * @param arg0
-   */
-  public HighlighterTest(String arg0) {
-    super(arg0);
-  }
+	/**
+	 * Constructor for HighlightExtractorTest.
+	 * 
+	 * @param arg0
+	 */
+	public HighlighterTest(String arg0) {
+		super(arg0);
+	}
 
-  public void testQueryScorerHits() throws Exception {
-    Analyzer analyzer = new SimpleAnalyzer();
-    QueryParser qp = new QueryParser(FIELD_NAME, analyzer);
-    query = qp.parse("\"very long\"");
-    searcher = new IndexSearcher(ramDir, false);
-    TopDocs hits = searcher.search(query, 10);
-    
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(scorer);
+	public void testQueryScorerHits() throws Exception {
+		Analyzer analyzer = new SimpleAnalyzer();
+		QueryParser qp = new QueryParser(FIELD_NAME, analyzer);
+		query = qp.parse("\"very long\"");
+		searcher = new IndexSearcher(ramDir, false);
+		TopDocs hits = searcher.search(query, 10);
 
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(scorer);
 
-    for (int i = 0; i < hits.scoreDocs.length; i++) {
-      Document doc = searcher.doc(hits.scoreDocs[i].doc);
-      String storedField = doc.get(FIELD_NAME);
+		for (int i = 0; i < hits.scoreDocs.length; i++) {
+			Document doc = searcher.doc(hits.scoreDocs[i].doc);
+			String storedField = doc.get(FIELD_NAME);
 
-      TokenStream stream = TokenSources.getAnyTokenStream(searcher
-          .getIndexReader(), hits.scoreDocs[i].doc, FIELD_NAME, doc, analyzer);
+			TokenStream stream = TokenSources.getAnyTokenStream(searcher
+					.getIndexReader(), hits.scoreDocs[i].doc, FIELD_NAME, doc,
+					analyzer);
 
-      Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
+			Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);
 
-      highlighter.setTextFragmenter(fragmenter);
+			highlighter.setTextFragmenter(fragmenter);
 
-      String fragment = highlighter.getBestFragment(stream, storedField);
+			String fragment = highlighter.getBestFragment(stream, storedField);
 
-      System.out.println(fragment);
-    }
-  }
-  
-  public void testHighlightingWithDefaultField() throws Exception {
+			System.out.println(fragment);
+		}
+	}
 
-    String s1 = "I call our world Flatland, not because we call it so,";
+	public void testHighlightingWithDefaultField() throws Exception {
 
-    QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer(Version.LUCENE_CURRENT));
+		String s1 = "I call our world Flatland, not because we call it so,";
 
-    // Verify that a query against the default field results in text being
-    // highlighted
-    // regardless of the field name.
-    Query q = parser.parse("\"world Flatland\"~3");
-    String expected = "I call our <B>world</B> <B>Flatland</B>, not because we call it so,";
-    String observed = highlightField(q, "SOME_FIELD_NAME", s1);
-    System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
-    assertEquals("Query in the default field results in text for *ANY* field being highlighted",
-        expected, observed);
+		QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer(
+				Version.LUCENE_CURRENT));
 
-    // Verify that a query against a named field does not result in any
-    // highlighting
-    // when the query field name differs from the name of the field being
-    // highlighted,
-    // which in this example happens to be the default field name.
-    q = parser.parse("text:\"world Flatland\"~3");
-    expected = s1;
-    observed = highlightField(q, FIELD_NAME, s1);
-    System.out.println("Expected: \"" + expected + "\n" + "Observed: \"" + observed);
-    assertEquals(
-        "Query in a named field does not result in highlighting when that field isn't in the query",
-        s1, highlightField(q, FIELD_NAME, s1));
-  }
+		// Verify that a query against the default field results in text being
+		// highlighted
+		// regardless of the field name.
+		Query q = parser.parse("\"world Flatland\"~3");
+		String expected = "I call our <B>world</B> <B>Flatland</B>, not because we call it so,";
+		String observed = highlightField(q, "SOME_FIELD_NAME", s1);
+		System.out.println("Expected: \"" + expected + "\n" + "Observed: \""
+				+ observed);
+		assertEquals(
+				"Query in the default field results in text for *ANY* field being highlighted",
+				expected, observed);
 
-  /**
-   * This method intended for use with <tt>testHighlightingWithDefaultField()</tt>
- * @throws InvalidTokenOffsetsException 
-   */
-  private static String highlightField(Query query, String fieldName, String text)
-      throws IOException, InvalidTokenOffsetsException {
-    TokenStream tokenStream = new StandardAnalyzer(Version.LUCENE_CURRENT).tokenStream(fieldName, new StringReader(text));
-    // Assuming "<B>", "</B>" used to highlight
-    SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
-    QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(formatter, scorer);
-    highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
+		// Verify that a query against a named field does not result in any
+		// highlighting
+		// when the query field name differs from the name of the field being
+		// highlighted,
+		// which in this example happens to be the default field name.
+		q = parser.parse("text:\"world Flatland\"~3");
+		expected = s1;
+		observed = highlightField(q, FIELD_NAME, s1);
+		System.out.println("Expected: \"" + expected + "\n" + "Observed: \""
+				+ observed);
+		assertEquals(
+				"Query in a named field does not result in highlighting when that field isn't in the query",
+				s1, highlightField(q, FIELD_NAME, s1));
+	}
 
-    String rv = highlighter.getBestFragments(tokenStream, text, 1, "(FIELD TEXT TRUNCATED)");
-    return rv.length() == 0 ? text : rv;
-  }
+	/*
+	 * TODO: Why is that second instance of the term "fish" highlighted? It is
+	 * not followed by the term "chips", so it should not be highlighted.
+	 */
+	public void testHighlightingNestedSpans() throws Exception {
 
-  public void testSimpleSpanHighlighter() throws Exception {
-    doSearching("Kennedy");
+		String pubText = "Sam dislikes most of the food and has to order"
+				+ " fish and chips - however the fish is frozen, not fresh.";
 
-    int maxNumFragmentsRequired = 2;
+		String fieldName = "SOME_FIELD_NAME";
 
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
-          new StringReader(text));
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+		SpanOrQuery spanOr = new SpanOrQuery(new SpanTermQuery[] {
+				new SpanTermQuery(new Term(fieldName, "fish")),
+				new SpanTermQuery(new Term(fieldName, "term1")),
+				new SpanTermQuery(new Term(fieldName, "term2")),
+				new SpanTermQuery(new Term(fieldName, "term3")) });
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
-    }
+		SpanNearQuery innerSpanNear = new SpanNearQuery(new SpanQuery[] {
+				spanOr, new SpanTermQuery(new Term(fieldName, "chips")) }, 2,
+				true);
 
-    // Not sure we can assert anything here - just running to check we dont
-    // throw any exceptions
-  }
+		SpanNearQuery query = new SpanNearQuery(
+				new SpanQuery[] { innerSpanNear,
+						new SpanTermQuery(new Term(fieldName, "frozen")) }, 8,
+				true);
 
-  // LUCENE-1752
-  public void testRepeatingTermsInMultBooleans() throws Exception {
-    String content = "x y z a b c d e f g b c g";
-    String ph1 = "\"a b c d\"";
-    String ph2 = "\"b c g\"";
-    String f1 = "f1";
-    String f2 = "f2";
-    String f1c = f1 + ":";
-    String f2c = f2 + ":";
-    String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2
-        + " OR " + f2c + ph2 + ")";
-    Analyzer analyzer = new WhitespaceAnalyzer();
-    QueryParser qp = new QueryParser(f1, analyzer);
-    Query query = qp.parse(q);
+		String expected = "Sam dislikes most of the food and has to order"
+				+ " <B>fish</B> and <B>chips</B> - however the fish is <B>frozen</B>, not fresh.";
+		String observed = highlightField(query, fieldName, pubText);
+		System.out.println("Expected: \"" + expected + "\n" + "Observed: \""
+				+ observed);
+		assertEquals(
+				"Why is that second instance of the term \"fish\" highlighted?",
+				expected, observed);
+	}
 
-    QueryScorer scorer = new QueryScorer(query, f1);
-    scorer.setExpandMultiTermQuery(false);
+	/*
+	 * TODO Ref: http://www.lucidimagination.com/blog/2009/07/18/the-spanquery/
+	 */
+	public void testHighlightingNestedSpans2() throws Exception {
 
-    Highlighter h = new Highlighter(this, scorer);
+		String pubText = "The Lucene was made by Doug Cutting and Lucene great Hadoop was"; // Problem
+		//String pubText = "The Lucene was made by Doug Cutting and the great Hadoop was"; // Works okay
 
-    h.getBestFragment(analyzer, f1, content);
+		String fieldName = "SOME_FIELD_NAME";
 
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 7);
-  }
+		SpanNearQuery spanNear = new SpanNearQuery(new SpanQuery[] {
+				new SpanTermQuery(new Term(fieldName, "lucene")),
+				new SpanTermQuery(new Term(fieldName, "doug")) }, 5, true);
 
-  public void testSimpleQueryScorerPhraseHighlighting() throws Exception {
-    doSearching("\"very long and contains\"");
+		Query query = new SpanNearQuery(new SpanQuery[] { spanNear,
+				new SpanTermQuery(new Term(fieldName, "hadoop")) }, 4, true);
 
-    int maxNumFragmentsRequired = 2;
+		String expected = "The <B>Lucene</B> was made by <B>Doug</B> Cutting and Lucene great <B>Hadoop</B> was";  // Problem
+		//String expected = "The <B>Lucene</B> was made by <B>Doug</B> Cutting and the great <B>Hadoop</B> was"; // Works okay
+		String observed = highlightField(query, fieldName, pubText);
+		System.out.println("Expected: \"" + expected + "\n" + "Observed: \""
+				+ observed);
+		assertEquals(
+				"Why is that second instance of the term \"Lucene\" highlighted?",
+				expected, observed);
+	}
+	
+	/*
+	 * TODO This is just to help visualize why this is a hit, via explain()
+	 */
+	public void testHighlightingNestedSpansExplanation() throws Exception {
+		//String pubText = "The Lucene was made by Doug Cutting and Lucene great Hadoop was";
+		//String pubText = "The Lucene was made by Doug Cutting and the great Hadoop was";
+		String pubText = "The Lucene Lucene Lucene Lucene Doug Cutting and the great Hadoop was";
+		String fieldName = "SOME_FIELD_NAME";
 
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(this, scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		SpanNearQuery spanNear = new SpanNearQuery(new SpanQuery[] {
+				new SpanTermQuery(new Term(fieldName, "lucene")),
+				new SpanTermQuery(new Term(fieldName, "doug")) }, 5, true);
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+		Query query = new SpanNearQuery(new SpanQuery[] { spanNear,
+				new SpanTermQuery(new Term(fieldName, "hadoop")) }, 4, true);
+		
+		RAMDirectory ramDir = new RAMDirectory();
+		IndexWriter writer = new IndexWriter(ramDir, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+		Document doc = new Document();
+		doc.add(new Field(fieldName, pubText, Field.Store.NO, Field.Index.ANALYZED));
+		writer.addDocument(doc);
+		writer.close();
+		IndexSearcher searcher = new IndexSearcher(ramDir, true);
+		Explanation expl = searcher.explain(query, 0);
+		System.err.println("Explanation: " + expl);
+		assertTrue(true);
+	}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
-    }
+	/**
+	 * This method intended for use with
+	 * <tt>testHighlightingWithDefaultField()</tt>
+	 * 
+	 * @throws InvalidTokenOffsetsException
+	 */
+	private static String highlightField(Query query, String fieldName,
+			String text) throws IOException, InvalidTokenOffsetsException {
+		TokenStream tokenStream = new StandardAnalyzer(Version.LUCENE_CURRENT)
+				.tokenStream(fieldName, new StringReader(text));
+		// Assuming "<B>", "</B>" used to highlight
+		SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
+		QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(formatter, scorer);
+		highlighter.setTextFragmenter(new SimpleFragmenter(Integer.MAX_VALUE));
 
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 3);
-  }
-  
-  public void testSpanRegexQuery() throws Exception {
-    query = new SpanOrQuery(new SpanQuery [] {
-        new SpanRegexQuery(new Term(FIELD_NAME, "ken.*")) });
-    searcher = new IndexSearcher(ramDir, true);
-    hits = searcher.search(query);
-    int maxNumFragmentsRequired = 2;
+		String rv = highlighter.getBestFragments(tokenStream, text, 1,
+				"(FIELD TEXT TRUNCATED)");
+		return rv.length() == 0 ? text : rv;
+	}
 
-    QueryScorer scorer = new QueryScorer(query, null);
-    Highlighter highlighter = new Highlighter(this, scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+	public void testSimpleSpanHighlighter() throws Exception {
+		doSearching("Kennedy");
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+		int maxNumFragmentsRequired = 2;
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
-    }
-    
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-  }
-  
-  public void testNumericRangeQuery() throws Exception {
-    // doesn't currently highlight, but make sure it doesn't cause exception either
-    query = NumericRangeQuery.newIntRange(NUMERIC_FIELD_NAME, new Integer(2), new Integer(6), true, true);
-    searcher = new IndexSearcher(ramDir);
-    hits = searcher.search(query);
-    int maxNumFragmentsRequired = 2;
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(scorer);
 
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(this, scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(NUMERIC_FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+		}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      //System.out.println("\t" + result);
-    }
+		// Not sure we can assert anything here - just running to check we dont
+		// throw any exceptions
+	}
 
+	// LUCENE-1752
+	public void testRepeatingTermsInMultBooleans() throws Exception {
+		String content = "x y z a b c d e f g b c g";
+		String ph1 = "\"a b c d\"";
+		String ph2 = "\"b c g\"";
+		String f1 = "f1";
+		String f2 = "f2";
+		String f1c = f1 + ":";
+		String f2c = f2 + ":";
+		String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2
+				+ " OR " + f2c + ph2 + ")";
+		Analyzer analyzer = new WhitespaceAnalyzer();
+		QueryParser qp = new QueryParser(f1, analyzer);
+		Query query = qp.parse(q);
 
-  }
+		QueryScorer scorer = new QueryScorer(query, f1);
+		scorer.setExpandMultiTermQuery(false);
 
-  public void testSimpleQueryScorerPhraseHighlighting2() throws Exception {
-    doSearching("\"text piece long\"~5");
+		Highlighter h = new Highlighter(this, scorer);
 
-    int maxNumFragmentsRequired = 2;
+		h.getBestFragment(analyzer, f1, content);
 
-    QueryScorer scorer =  new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(this,scorer);
-    highlighter.setTextFragmenter(new SimpleFragmenter(40));
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 7);
+	}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
-    }
+	public void testSimpleQueryScorerPhraseHighlighting() throws Exception {
+		doSearching("\"very long and contains\"");
 
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 6);
-  }
+		int maxNumFragmentsRequired = 2;
 
-  public void testSimpleQueryScorerPhraseHighlighting3() throws Exception {
-    doSearching("\"x y z\"");
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(this, scorer);
 
-    int maxNumFragmentsRequired = 2;
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-      QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-      Highlighter highlighter = new Highlighter(this, scorer);
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+		}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 3);
+	}
 
-      assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-          numHighlights == 3);
-    }
-  }
-  
-  public void testSimpleSpanFragmenter() throws Exception {
-    doSearching("\"piece of text that is very long\"");
+	public void testSpanRegexQuery() throws Exception {
+		query = new SpanOrQuery(new SpanQuery[] { new SpanRegexQuery(new Term(
+				FIELD_NAME, "ken.*")) });
+		searcher = new IndexSearcher(ramDir, true);
+		hits = searcher.search(query);
+		int maxNumFragmentsRequired = 2;
 
-    int maxNumFragmentsRequired = 2;
+		QueryScorer scorer = new QueryScorer(query, null);
+		Highlighter highlighter = new Highlighter(this, scorer);
 
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(this, scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-      highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-      String result = highlighter.getBestFragments(tokenStream, text,
-          maxNumFragmentsRequired, "...");
-      System.out.println("\t" + result);
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+		}
 
-    }
-    
-    doSearching("\"been shot\"");
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
+	}
 
-    maxNumFragmentsRequired = 2;
-    
-    scorer = new QueryScorer(query, FIELD_NAME);
-    highlighter = new Highlighter(this, scorer);
+	public void testNumericRangeQuery() throws Exception {
+		// doesn't currently highlight, but make sure it doesn't cause exception
+		// either
+		query = NumericRangeQuery.newIntRange(NUMERIC_FIELD_NAME,
+				new Integer(2), new Integer(6), true, true);
+		searcher = new IndexSearcher(ramDir);
+		hits = searcher.search(query);
+		int maxNumFragmentsRequired = 2;
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(this, scorer);
 
-      highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(NUMERIC_FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-      String result = highlighter.getBestFragments(tokenStream, text,
-          maxNumFragmentsRequired, "...");
-      System.out.println("\t" + result);
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-    }
-  }
-  
-  // position sensitive query added after position insensitive query
-  public void testPosTermStdTerm() throws Exception {
-    doSearching("y \"x y z\"");
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			// System.out.println("\t" + result);
+		}
 
-    int maxNumFragmentsRequired = 2;
-    
-    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-    Highlighter highlighter = new Highlighter(this,scorer);
-    
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,new StringReader(text));
+	}
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+	public void testSimpleQueryScorerPhraseHighlighting2() throws Exception {
+		doSearching("\"text piece long\"~5");
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
+		int maxNumFragmentsRequired = 2;
 
-      assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-          numHighlights == 4);
-    }
-  }
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(this, scorer);
+		highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-  public void testQueryScorerMultiPhraseQueryHighlighting() throws Exception {
-    MultiPhraseQuery mpq = new MultiPhraseQuery();
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-    mpq.add(new Term[] { new Term(FIELD_NAME, "wordx"), new Term(FIELD_NAME, "wordb") });
-    mpq.add(new Term(FIELD_NAME, "wordy"));
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+		}
 
-    doSearching(mpq);
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 6);
+	}
 
-    final int maxNumFragmentsRequired = 2;
-    assertExpectedHighlightCount(maxNumFragmentsRequired, 6);
-  }
+	public void testSimpleQueryScorerPhraseHighlighting3() throws Exception {
+		doSearching("\"x y z\"");
 
-  public void testQueryScorerMultiPhraseQueryHighlightingWithGap() throws Exception {
-    MultiPhraseQuery mpq = new MultiPhraseQuery();
+		int maxNumFragmentsRequired = 2;
 
-    /*
-     * The toString of MultiPhraseQuery doesn't work so well with these
-     * out-of-order additions, but the Query itself seems to match accurately.
-     */
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
+			QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-    mpq.add(new Term[] { new Term(FIELD_NAME, "wordz") }, 2);
-    mpq.add(new Term[] { new Term(FIELD_NAME, "wordx") }, 0);
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-    doSearching(mpq);
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
 
-    final int maxNumFragmentsRequired = 1;
-    final int expectedHighlights = 2;
+			assertTrue("Failed to find correct number of highlights "
+					+ numHighlights + " found", numHighlights == 3);
+		}
+	}
 
-    assertExpectedHighlightCount(maxNumFragmentsRequired, expectedHighlights);
-  }
+	public void testSimpleSpanFragmenter() throws Exception {
+		doSearching("\"piece of text that is very long\"");
 
-  public void testNearSpanSimpleQuery() throws Exception {
-    doSearching(new SpanNearQuery(new SpanQuery[] {
-        new SpanTermQuery(new Term(FIELD_NAME, "beginning")),
-        new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false));
+		int maxNumFragmentsRequired = 2;
 
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(this, scorer);
 
-      public void run() throws Exception {
-        mode = QUERY;
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-      }
-    };
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-    helper.run();
+			highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 5));
 
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 2);
-  }
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
 
-  public void testSimpleQueryTermScorerHighlighter() throws Exception {
-    doSearching("Kennedy");
-    Highlighter highlighter = new Highlighter(new QueryTermScorer(query));
-    highlighter.setTextFragmenter(new SimpleFragmenter(40));
-    int maxNumFragmentsRequired = 2;
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
-    }
-    // Not sure we can assert anything here - just running to check we dont
-    // throw any exceptions
-  }
+		doSearching("\"been shot\"");
 
-  public void testSpanHighlighting() throws Exception {
-    Query query1 = new SpanNearQuery(new SpanQuery[] {
-        new SpanTermQuery(new Term(FIELD_NAME, "wordx")),
-        new SpanTermQuery(new Term(FIELD_NAME, "wordy")) }, 1, false);
-    Query query2 = new SpanNearQuery(new SpanQuery[] {
-        new SpanTermQuery(new Term(FIELD_NAME, "wordy")),
-        new SpanTermQuery(new Term(FIELD_NAME, "wordc")) }, 1, false);
-    BooleanQuery bquery = new BooleanQuery();
-    bquery.add(query1, Occur.SHOULD);
-    bquery.add(query2, Occur.SHOULD);
-    doSearching(bquery);
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		maxNumFragmentsRequired = 2;
 
-      public void run() throws Exception {
-        mode = QUERY;
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-      }
-    };
+		scorer = new QueryScorer(query, FIELD_NAME);
+		highlighter = new Highlighter(this, scorer);
 
-    helper.run();
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 7);
-  }
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-  public void testNotSpanSimpleQuery() throws Exception {
-    doSearching(new SpanNotQuery(new SpanNearQuery(new SpanQuery[] {
-        new SpanTermQuery(new Term(FIELD_NAME, "shot")),
-        new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false), new SpanTermQuery(
-        new Term(FIELD_NAME, "john"))));
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			highlighter.setTextFragmenter(new SimpleSpanFragmenter(scorer, 20));
 
-      public void run() throws Exception {
-        mode = QUERY;
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-      }
-    };
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
 
-    helper.run();
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 4);
-  }
+		}
+	}
 
-  public void testGetBestFragmentsSimpleQuery() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+	// position sensitive query added after position insensitive query
+	public void testPosTermStdTerm() throws Exception {
+		doSearching("y \"x y z\"");
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("Kennedy");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
-      }
-    };
+		int maxNumFragmentsRequired = 2;
 
-    helper.start();
-  }
+		QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+		Highlighter highlighter = new Highlighter(this, scorer);
 
-  public void testGetFuzzyFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("Kinnedy~");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this, true);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 5);
-      }
-    };
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-    helper.start();
-  }
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
 
-  public void testGetWildCardFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			assertTrue("Failed to find correct number of highlights "
+					+ numHighlights + " found", numHighlights == 4);
+		}
+	}
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("K?nnedy");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
-      }
-    };
+	public void testQueryScorerMultiPhraseQueryHighlighting() throws Exception {
+		MultiPhraseQuery mpq = new MultiPhraseQuery();
 
-    helper.start();
-  }
+		mpq.add(new Term[] { new Term(FIELD_NAME, "wordx"),
+				new Term(FIELD_NAME, "wordb") });
+		mpq.add(new Term(FIELD_NAME, "wordy"));
 
-  public void testGetMidWildCardFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		doSearching(mpq);
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("K*dy");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 5);
-      }
-    };
+		final int maxNumFragmentsRequired = 2;
+		assertExpectedHighlightCount(maxNumFragmentsRequired, 6);
+	}
 
-    helper.start();
-  }
+	public void testQueryScorerMultiPhraseQueryHighlightingWithGap()
+			throws Exception {
+		MultiPhraseQuery mpq = new MultiPhraseQuery();
 
-  public void testGetRangeFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		/*
+		 * The toString of MultiPhraseQuery doesn't work so well with these
+		 * out-of-order additions, but the Query itself seems to match
+		 * accurately.
+		 */
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        String queryString = FIELD_NAME + ":[kannedy TO kznnedy]";
+		mpq.add(new Term[] { new Term(FIELD_NAME, "wordz") }, 2);
+		mpq.add(new Term[] { new Term(FIELD_NAME, "wordx") }, 0);
 
-        // Need to explicitly set the QueryParser property to use TermRangeQuery
-        // rather
-        // than RangeFilters
-        QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer());
-        parser.setUseOldRangeQuery(true);
-        query = parser.parse(queryString);
-        doSearching(query);
+		doSearching(mpq);
 
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 5);
-      }
-    };
+		final int maxNumFragmentsRequired = 1;
+		final int expectedHighlights = 2;
 
-    helper.start();
-  }
+		assertExpectedHighlightCount(maxNumFragmentsRequired,
+				expectedHighlights);
+	}
 
-  public void testGetConstantScoreRangeFragments() throws Exception {
+	public void testNearSpanSimpleQuery() throws Exception {
+		doSearching(new SpanNearQuery(new SpanQuery[] {
+				new SpanTermQuery(new Term(FIELD_NAME, "beginning")),
+				new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) }, 3, false));
 
-    numHighlights = 0;
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    query = new ConstantScoreRangeQuery(FIELD_NAME, "kannedy", "kznnedy", true, true);
+			public void run() throws Exception {
+				mode = QUERY;
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+			}
+		};
 
-    searcher = new IndexSearcher(ramDir);
-    // can't rewrite ConstantScoreRangeQuery if you want to highlight it -
-    // it rewrites to ConstantScoreQuery which cannot be highlighted
-    // query = unReWrittenQuery.rewrite(reader);
-    System.out.println("Searching for: " + query.toString(FIELD_NAME));
-    hits = searcher.search(query);
+		helper.run();
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-      int maxNumFragmentsRequired = 2;
-      String fragmentSeparator = "...";
-      QueryScorer scorer = null;
-      TokenStream tokenStream = null;
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 2);
+	}
 
-      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-      
-      scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
+	public void testSimpleQueryTermScorerHighlighter() throws Exception {
+		doSearching("Kennedy");
+		Highlighter highlighter = new Highlighter(new QueryTermScorer(query));
+		highlighter.setTextFragmenter(new SimpleFragmenter(40));
+		int maxNumFragmentsRequired = 2;
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
 
-      Highlighter highlighter = new Highlighter(this, scorer);
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+		}
+		// Not sure we can assert anything here - just running to check we dont
+		// throw any exceptions
+	}
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(20));
+	public void testSpanHighlighting() throws Exception {
+		Query query1 = new SpanNearQuery(new SpanQuery[] {
+				new SpanTermQuery(new Term(FIELD_NAME, "wordx")),
+				new SpanTermQuery(new Term(FIELD_NAME, "wordy")) }, 1, false);
+		Query query2 = new SpanNearQuery(new SpanQuery[] {
+				new SpanTermQuery(new Term(FIELD_NAME, "wordy")),
+				new SpanTermQuery(new Term(FIELD_NAME, "wordc")) }, 1, false);
+		BooleanQuery bquery = new BooleanQuery();
+		bquery.add(query1, Occur.SHOULD);
+		bquery.add(query2, Occur.SHOULD);
+		doSearching(bquery);
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          fragmentSeparator);
-      System.out.println("\t" + result);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-  }
-  
-  public void testRangeQuery() throws Exception {
+			public void run() throws Exception {
+				mode = QUERY;
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+			}
+		};
 
-    numHighlights = 0;
+		helper.run();
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 7);
+	}
 
-    query = new RangeQuery(new Term(FIELD_NAME, "kannedy"), new Term(FIELD_NAME, "kznnedy"), true);
+	public void testNotSpanSimpleQuery() throws Exception {
+		doSearching(new SpanNotQuery(
+				new SpanNearQuery(new SpanQuery[] {
+						new SpanTermQuery(new Term(FIELD_NAME, "shot")),
+						new SpanTermQuery(new Term(FIELD_NAME, "kennedy")) },
+						3, false), new SpanTermQuery(new Term(FIELD_NAME,
+						"john"))));
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    searcher = new IndexSearcher(ramDir);
-    // can't rewrite ConstantScoreRangeQuery if you want to highlight it -
-    // it rewrites to ConstantScoreQuery which cannot be highlighted
-    // query = unReWrittenQuery.rewrite(reader);
-    System.out.println("Searching for: " + query.toString(FIELD_NAME));
-    hits = searcher.search(query);
+			public void run() throws Exception {
+				mode = QUERY;
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+			}
+		};
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-      int maxNumFragmentsRequired = 2;
-      String fragmentSeparator = "...";
-      QueryScorer scorer = null;
-      TokenStream tokenStream = null;
+		helper.run();
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 4);
+	}
 
-      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-      
-      scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
+	public void testGetBestFragmentsSimpleQuery() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-      Highlighter highlighter = new Highlighter(this, scorer);
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("Kennedy");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
+			}
+		};
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(20));
+		helper.start();
+	}
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          fragmentSeparator);
-      System.out.println("\t" + result);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-  }
-  
-  public void testConstantScoreMultiTermQuery() throws Exception {
+	public void testGetFuzzyFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    numHighlights = 0;
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("Kinnedy~");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this, true);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 5);
+			}
+		};
 
-    query = new WildcardQuery(new Term(FIELD_NAME, "ken*"));
-    ((WildcardQuery)query).setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
-    searcher = new IndexSearcher(ramDir);
-    // can't rewrite ConstantScore if you want to highlight it -
-    // it rewrites to ConstantScoreQuery which cannot be highlighted
-    // query = unReWrittenQuery.rewrite(reader);
-    System.out.println("Searching for: " + query.toString(FIELD_NAME));
-    hits = searcher.search(query);
+		helper.start();
+	}
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-      int maxNumFragmentsRequired = 2;
-      String fragmentSeparator = "...";
-      QueryScorer scorer = null;
-      TokenStream tokenStream = null;
+	public void testGetWildCardFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-      
-      scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("K?nnedy");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
+			}
+		};
 
-      Highlighter highlighter = new Highlighter(this, scorer);
+		helper.start();
+	}
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(20));
+	public void testGetMidWildCardFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          fragmentSeparator);
-      System.out.println("\t" + result);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-    
-    // try null field
-    
-    hits = searcher.search(query);
-    
-    numHighlights = 0;
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("K*dy");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 5);
+			}
+		};
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-      int maxNumFragmentsRequired = 2;
-      String fragmentSeparator = "...";
-      QueryScorer scorer = null;
-      TokenStream tokenStream = null;
+		helper.start();
+	}
 
-      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-      
-      scorer = new QueryScorer(query, null);
+	public void testGetRangeFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-      Highlighter highlighter = new Highlighter(this, scorer);
+			public void run() throws Exception {
+				numHighlights = 0;
+				String queryString = FIELD_NAME + ":[kannedy TO kznnedy]";
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(20));
+				// Need to explicitly set the QueryParser property to use
+				// TermRangeQuery
+				// rather
+				// than RangeFilters
+				QueryParser parser = new QueryParser(FIELD_NAME,
+						new StandardAnalyzer());
+				parser.setUseOldRangeQuery(true);
+				query = parser.parse(queryString);
+				doSearching(query);
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          fragmentSeparator);
-      System.out.println("\t" + result);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-    
-    // try default field
-    
-    hits = searcher.search(query);
-    
-    numHighlights = 0;
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 5);
+			}
+		};
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-      int maxNumFragmentsRequired = 2;
-      String fragmentSeparator = "...";
-      QueryScorer scorer = null;
-      TokenStream tokenStream = null;
+		helper.start();
+	}
 
-      tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-      
-      scorer = new QueryScorer(query, "random_field", HighlighterTest.FIELD_NAME);
+	public void testGetConstantScoreRangeFragments() throws Exception {
 
-      Highlighter highlighter = new Highlighter(this, scorer);
+		numHighlights = 0;
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(20));
+		query = new ConstantScoreRangeQuery(FIELD_NAME, "kannedy", "kznnedy",
+				true, true);
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          fragmentSeparator);
-      System.out.println("\t" + result);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 5);
-  }
+		searcher = new IndexSearcher(ramDir);
+		// can't rewrite ConstantScoreRangeQuery if you want to highlight it -
+		// it rewrites to ConstantScoreQuery which cannot be highlighted
+		// query = unReWrittenQuery.rewrite(reader);
+		System.out.println("Searching for: " + query.toString(FIELD_NAME));
+		hits = searcher.search(query);
 
-  public void testGetBestFragmentsPhrase() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+			int maxNumFragmentsRequired = 2;
+			String fragmentSeparator = "...";
+			QueryScorer scorer = null;
+			TokenStream tokenStream = null;
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("\"John Kennedy\"");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        // Currently highlights "John" and "Kennedy" separately
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 2);
-      }
-    };
+			tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME,
+					new StringReader(text));
 
-    helper.start();
-  }
+			scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
-  public void testGetBestFragmentsQueryScorer() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),
-            new SpanTermQuery(new Term("contents", "kennedy")), };
+			highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
-        SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
-        doSearching(snq);
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        // Currently highlights "John" and "Kennedy" separately
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 2);
-      }
-    };
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, fragmentSeparator);
+			System.out.println("\t" + result);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
+	}
 
-    helper.start();
-  }
+	public void testRangeQuery() throws Exception {
 
-  public void testOffByOne() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		numHighlights = 0;
 
-      public void run() throws Exception {
-        TermQuery query = new TermQuery(new Term("data", "help"));
-        Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryTermScorer(query));
-        hg.setTextFragmenter(new NullFragmenter());
+		query = new RangeQuery(new Term(FIELD_NAME, "kannedy"), new Term(
+				FIELD_NAME, "kznnedy"), true);
 
-        String match = null;
-        match = hg.getBestFragment(new StandardAnalyzer(), "data", "help me [54-65]");
-        assertEquals("<B>help</B> me [54-65]", match);
+		searcher = new IndexSearcher(ramDir);
+		// can't rewrite ConstantScoreRangeQuery if you want to highlight it -
+		// it rewrites to ConstantScoreQuery which cannot be highlighted
+		// query = unReWrittenQuery.rewrite(reader);
+		System.out.println("Searching for: " + query.toString(FIELD_NAME));
+		hits = searcher.search(query);
 
-      }
-    };
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+			int maxNumFragmentsRequired = 2;
+			String fragmentSeparator = "...";
+			QueryScorer scorer = null;
+			TokenStream tokenStream = null;
 
-    helper.start();
-  }
+			tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME,
+					new StringReader(text));
 
-  public void testGetBestFragmentsFilteredQuery() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        TermRangeFilter rf = new TermRangeFilter("contents", "john", "john", true, true);
-        SpanQuery clauses[] = { new SpanTermQuery(new Term("contents", "john")),
-            new SpanTermQuery(new Term("contents", "kennedy")), };
-        SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
-        FilteredQuery fq = new FilteredQuery(snq, rf);
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-        doSearching(fq);
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        // Currently highlights "John" and "Kennedy" separately
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 2);
-      }
-    };
+			highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
-    helper.start();
-  }
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, fragmentSeparator);
+			System.out.println("\t" + result);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
+	}
 
-  public void testGetBestFragmentsFilteredPhraseQuery() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+	public void testConstantScoreMultiTermQuery() throws Exception {
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        TermRangeFilter rf = new TermRangeFilter("contents", "john", "john", true, true);
-        PhraseQuery pq = new PhraseQuery();
-        pq.add(new Term("contents", "john"));
-        pq.add(new Term("contents", "kennedy"));
-        FilteredQuery fq = new FilteredQuery(pq, rf);
+		numHighlights = 0;
 
-        doSearching(fq);
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        // Currently highlights "John" and "Kennedy" separately
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 2);
-      }
-    };
+		query = new WildcardQuery(new Term(FIELD_NAME, "ken*"));
+		((WildcardQuery) query)
+				.setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE);
+		searcher = new IndexSearcher(ramDir);
+		// can't rewrite ConstantScore if you want to highlight it -
+		// it rewrites to ConstantScoreQuery which cannot be highlighted
+		// query = unReWrittenQuery.rewrite(reader);
+		System.out.println("Searching for: " + query.toString(FIELD_NAME));
+		hits = searcher.search(query);
 
-    helper.start();
-  }
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+			int maxNumFragmentsRequired = 2;
+			String fragmentSeparator = "...";
+			QueryScorer scorer = null;
+			TokenStream tokenStream = null;
 
-  public void testGetBestFragmentsMultiTerm() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME,
+					new StringReader(text));
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("John Kenn*");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 5);
-      }
-    };
+			scorer = new QueryScorer(query, HighlighterTest.FIELD_NAME);
 
-    helper.start();
-  }
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-  public void testGetBestFragmentsWithOr() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("JFK OR Kennedy");
-        doStandardHighlights(analyzer, hits, query, HighlighterTest.this);
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 5);
-      }
-    };
-    helper.start();
-  }
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, fragmentSeparator);
+			System.out.println("\t" + result);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
 
-  public void testGetBestSingleFragment() throws Exception {
+		// try null field
 
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		hits = searcher.search(query);
 
-      public void run() throws Exception {
-        doSearching("Kennedy");
-        numHighlights = 0;
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+		numHighlights = 0;
 
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
-          highlighter.setTextFragmenter(new SimpleFragmenter(40));
-          String result = highlighter.getBestFragment(tokenStream, text);
-          System.out.println("\t" + result);
-        }
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+			int maxNumFragmentsRequired = 2;
+			String fragmentSeparator = "...";
+			QueryScorer scorer = null;
+			TokenStream tokenStream = null;
 
-        numHighlights = 0;
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
-          highlighter.getBestFragment(analyzer, FIELD_NAME, text);
-        }
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
+			tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME,
+					new StringReader(text));
 
-        numHighlights = 0;
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
+			scorer = new QueryScorer(query, null);
 
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
-          highlighter.getBestFragments(analyzer, FIELD_NAME, text, 10);
-        }
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-      }
+			highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
-    };
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, fragmentSeparator);
+			System.out.println("\t" + result);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
 
-    helper.start();
+		// try default field
 
-  }
+		hits = searcher.search(query);
 
-  public void testGetBestSingleFragmentWithWeights() throws Exception {
+		numHighlights = 0;
 
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+			int maxNumFragmentsRequired = 2;
+			String fragmentSeparator = "...";
+			QueryScorer scorer = null;
+			TokenStream tokenStream = null;
 
-      public void run() throws Exception {
-        WeightedSpanTerm[] wTerms = new WeightedSpanTerm[2];
-        wTerms[0] = new WeightedSpanTerm(10f, "hello");
+			tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME,
+					new StringReader(text));
 
-        List positionSpans = new ArrayList();
-        positionSpans.add(new PositionSpan(0, 0));
-        wTerms[0].addPositionSpans(positionSpans);
+			scorer = new QueryScorer(query, "random_field",
+					HighlighterTest.FIELD_NAME);
 
-        wTerms[1] = new WeightedSpanTerm(1f, "kennedy");
-        positionSpans = new ArrayList();
-        positionSpans.add(new PositionSpan(14, 14));
-        wTerms[1].addPositionSpans(positionSpans);
+			Highlighter highlighter = new Highlighter(this, scorer);
 
-        Highlighter highlighter = getHighlighter(wTerms, HighlighterTest.this);// new
-        // Highlighter(new
-        // QueryTermScorer(wTerms));
-        TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
-        highlighter.setTextFragmenter(new SimpleFragmenter(2));
+			highlighter.setTextFragmenter(new SimpleFragmenter(20));
 
-        String result = highlighter.getBestFragment(tokenStream, texts[0]).trim();
-        assertTrue("Failed to find best section using weighted terms. Found: [" + result + "]",
-            "<B>Hello</B>".equals(result));
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, fragmentSeparator);
+			System.out.println("\t" + result);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 5);
+	}
 
-        // readjust weights
-        wTerms[1].setWeight(50f);
-        tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
-        highlighter = getHighlighter(wTerms, HighlighterTest.this);
-        highlighter.setTextFragmenter(new SimpleFragmenter(2));
+	public void testGetBestFragmentsPhrase() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        result = highlighter.getBestFragment(tokenStream, texts[0]).trim();
-        assertTrue("Failed to find best section using weighted terms. Found: " + result,
-            "<B>kennedy</B>".equals(result));
-      }
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("\"John Kennedy\"");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				// Currently highlights "John" and "Kennedy" separately
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 2);
+			}
+		};
 
-    };
+		helper.start();
+	}
 
-    helper.start();
+	public void testGetBestFragmentsQueryScorer() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-  }
+			public void run() throws Exception {
+				numHighlights = 0;
+				SpanQuery clauses[] = {
+						new SpanTermQuery(new Term("contents", "john")),
+						new SpanTermQuery(new Term("contents", "kennedy")), };
 
-  // tests a "complex" analyzer that produces multiple
-  // overlapping tokens
-  public void testOverlapAnalyzer() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+				SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
+				doSearching(snq);
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				// Currently highlights "John" and "Kennedy" separately
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 2);
+			}
+		};
 
-      public void run() throws Exception {
-        HashMap synonyms = new HashMap();
-        synonyms.put("football", "soccer,footie");
-        Analyzer analyzer = new SynonymAnalyzer(synonyms);
-        String srchkey = "football";
+		helper.start();
+	}
 
-        String s = "football-soccer in the euro 2004 footie competition";
-        QueryParser parser = new QueryParser("bookid", analyzer);
-        Query query = parser.parse(srchkey);
+	public void testOffByOne() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        TokenStream tokenStream = analyzer.tokenStream(null, new StringReader(s));
+			public void run() throws Exception {
+				TermQuery query = new TermQuery(new Term("data", "help"));
+				Highlighter hg = new Highlighter(new SimpleHTMLFormatter(),
+						new QueryTermScorer(query));
+				hg.setTextFragmenter(new NullFragmenter());
 
-        Highlighter highlighter = getHighlighter(query, null, tokenStream, HighlighterTest.this);
+				String match = null;
+				match = hg.getBestFragment(new StandardAnalyzer(), "data",
+						"help me [54-65]");
+				assertEquals("<B>help</B> me [54-65]", match);
 
-        // Get 3 best fragments and seperate with a "..."
-        tokenStream = analyzer.tokenStream(null, new StringReader(s));
+			}
+		};
 
-        String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
-        String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
-        assertTrue("overlapping analyzer should handle highlights OK, expected:" + expectedResult
-            + " actual:" + result, expectedResult.equals(result));
-      }
+		helper.start();
+	}
 
-    };
+	public void testGetBestFragmentsFilteredQuery() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    helper.start();
+			public void run() throws Exception {
+				numHighlights = 0;
+				TermRangeFilter rf = new TermRangeFilter("contents", "john",
+						"john", true, true);
+				SpanQuery clauses[] = {
+						new SpanTermQuery(new Term("contents", "john")),
+						new SpanTermQuery(new Term("contents", "kennedy")), };
+				SpanNearQuery snq = new SpanNearQuery(clauses, 1, true);
+				FilteredQuery fq = new FilteredQuery(snq, rf);
 
-  }
+				doSearching(fq);
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				// Currently highlights "John" and "Kennedy" separately
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 2);
+			}
+		};
 
-  public void testGetSimpleHighlight() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		helper.start();
+	}
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("Kennedy");
-        // new Highlighter(HighlighterTest.this, new QueryTermScorer(query));
+	public void testGetBestFragmentsFilteredPhraseQuery() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
-          String result = highlighter.getBestFragment(tokenStream, text);
-          System.out.println("\t" + result);
-        }
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 4);
-      }
-    };
-    helper.start();
-  }
+			public void run() throws Exception {
+				numHighlights = 0;
+				TermRangeFilter rf = new TermRangeFilter("contents", "john",
+						"john", true, true);
+				PhraseQuery pq = new PhraseQuery();
+				pq.add(new Term("contents", "john"));
+				pq.add(new Term("contents", "kennedy"));
+				FilteredQuery fq = new FilteredQuery(pq, rf);
 
-  public void testGetTextFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+				doSearching(fq);
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				// Currently highlights "John" and "Kennedy" separately
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 2);
+			}
+		};
 
-      public void run() throws Exception {
+		helper.start();
+	}
 
-        doSearching("Kennedy");
+	public void testGetBestFragmentsMultiTerm() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("John Kenn*");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 5);
+			}
+		};
 
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);// new Highlighter(this, new
-          // QueryTermScorer(query));
-          highlighter.setTextFragmenter(new SimpleFragmenter(20));
-          String stringResults[] = highlighter.getBestFragments(tokenStream, text, 10);
+		helper.start();
+	}
 
-          tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          TextFragment fragmentResults[] = highlighter.getBestTextFragments(tokenStream, text,
-              true, 10);
+	public void testGetBestFragmentsWithOr() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-          assertTrue("Failed to find correct number of text Fragments: " + fragmentResults.length
-              + " vs " + stringResults.length, fragmentResults.length == stringResults.length);
-          for (int j = 0; j < stringResults.length; j++) {
-            System.out.println(fragmentResults[j]);
-            assertTrue("Failed to find same text Fragments: " + fragmentResults[j] + " found",
-                fragmentResults[j].toString().equals(stringResults[j]));
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("JFK OR Kennedy");
+				doStandardHighlights(analyzer, hits, query,
+						HighlighterTest.this);
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 5);
+			}
+		};
+		helper.start();
+	}
 
-          }
+	public void testGetBestSingleFragment() throws Exception {
 
-        }
-      }
-    };
-    helper.start();
-  }
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-  public void testMaxSizeHighlight() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			public void run() throws Exception {
+				doSearching("Kennedy");
+				numHighlights = 0;
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        doSearching("meat");
-        TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(texts[0]));
-        Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-            HighlighterTest.this);// new Highlighter(this, new
-        // QueryTermScorer(query));
-        highlighter.setMaxDocBytesToAnalyze(30);
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);
+					highlighter.setTextFragmenter(new SimpleFragmenter(40));
+					String result = highlighter.getBestFragment(tokenStream,
+							text);
+					System.out.println("\t" + result);
+				}
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
 
-        highlighter.getBestFragment(tokenStream, texts[0]);
-        assertTrue("Setting MaxDocBytesToAnalyze should have prevented "
-            + "us from finding matches for this record: " + numHighlights + " found",
-            numHighlights == 0);
-      }
-    };
+				numHighlights = 0;
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);
+					highlighter.getBestFragment(analyzer, FIELD_NAME, text);
+				}
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
 
-    helper.start();
-  }
+				numHighlights = 0;
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
 
-  public void testMaxSizeHighlightTruncates() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);
+					highlighter
+							.getBestFragments(analyzer, FIELD_NAME, text, 10);
+				}
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
 
-      public void run() throws Exception {
-        String goodWord = "goodtoken";
-        Set stopWords = new HashSet(1);
-        stopWords.add("stoppedtoken");
+			}
 
-        TermQuery query = new TermQuery(new Term("data", goodWord));
+		};
 
-        String match = null;
-        StringBuffer sb = new StringBuffer();
-        sb.append(goodWord);
-        for (int i = 0; i < 10000; i++) {
-          sb.append(" ");
-          // only one stopword
-          sb.append(stopWords.iterator().next());
-        }
-        SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
-        Highlighter hg = getHighlighter(query, "data", new StandardAnalyzer(stopWords).tokenStream(
-            "data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
-        // new
-        // QueryTermScorer(query));
-        hg.setTextFragmenter(new NullFragmenter());
-        hg.setMaxDocBytesToAnalyze(100);
-        match = hg.getBestFragment(new StandardAnalyzer(stopWords), "data", sb.toString());
-        assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
-            .getMaxDocBytesToAnalyze());
+		helper.start();
 
-        // add another tokenized word to the overrall length - but set way
-        // beyond
-        // the length of text under consideration (after a large slug of stop
-        // words
-        // + whitespace)
-        sb.append(" ");
-        sb.append(goodWord);
-        match = hg.getBestFragment(new StandardAnalyzer(stopWords), "data", sb.toString());
-        assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
-            .getMaxDocBytesToAnalyze());
-      }
-    };
+	}
 
-    helper.start();
+	public void testGetBestSingleFragmentWithWeights() throws Exception {
 
-  }
-  
-  public void testMaxSizeEndHighlight() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
-      public void run() throws Exception {
-        Set stopWords = new HashSet();
-        stopWords.add("in");
-        stopWords.add("it");
-        TermQuery query = new TermQuery(new Term("text", "searchterm"));
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        String text = "this is a text with searchterm in it";
-        SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
-        Highlighter hg = getHighlighter(query, "text", new StandardAnalyzer(
-            stopWords).tokenStream("text", new StringReader(text)), fm);
-        hg.setTextFragmenter(new NullFragmenter());
-        hg.setMaxDocCharsToAnalyze(36);
-        String match = hg.getBestFragment(new StandardAnalyzer(stopWords), "text", text);
-        assertTrue(
-            "Matched text should contain remainder of text after highlighted query ",
-            match.endsWith("in it"));
-      }
-    };
-    helper.start();
-  }
+			public void run() throws Exception {
+				WeightedSpanTerm[] wTerms = new WeightedSpanTerm[2];
+				wTerms[0] = new WeightedSpanTerm(10f, "hello");
 
-  public void testUnRewrittenQuery() throws Exception {
-    final TestHighlightRunner helper = new TestHighlightRunner() {
+				List positionSpans = new ArrayList();
+				positionSpans.add(new PositionSpan(0, 0));
+				wTerms[0].addPositionSpans(positionSpans);
 
-      public void run() throws Exception {
-        numHighlights = 0;
-        // test to show how rewritten query can still be used
-        searcher = new IndexSearcher(ramDir);
-        Analyzer analyzer = new StandardAnalyzer();
+				wTerms[1] = new WeightedSpanTerm(1f, "kennedy");
+				positionSpans = new ArrayList();
+				positionSpans.add(new PositionSpan(14, 14));
+				wTerms[1].addPositionSpans(positionSpans);
 
-        QueryParser parser = new QueryParser(FIELD_NAME, analyzer);
-        Query query = parser.parse("JF? or Kenned*");
-        System.out.println("Searching with primitive query");
-        // forget to set this and...
-        // query=query.rewrite(reader);
-        Hits hits = searcher.search(query);
+				Highlighter highlighter = getHighlighter(wTerms,
+						HighlighterTest.this);// new
+				// Highlighter(new
+				// QueryTermScorer(wTerms));
+				TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+						new StringReader(texts[0]));
+				highlighter.setTextFragmenter(new SimpleFragmenter(2));
 
-        // create an instance of the highlighter with the tags used to surround
-        // highlighted text
-        // QueryHighlightExtractor highlighter = new
-        // QueryHighlightExtractor(this,
-        // query, new StandardAnalyzer());
+				String result = highlighter.getBestFragment(tokenStream,
+						texts[0]).trim();
+				assertTrue(
+						"Failed to find best section using weighted terms. Found: ["
+								+ result + "]", "<B>Hello</B>".equals(result));
 
-        int maxNumFragmentsRequired = 3;
+				// readjust weights
+				wTerms[1].setWeight(50f);
+				tokenStream = analyzer.tokenStream(FIELD_NAME,
+						new StringReader(texts[0]));
+				highlighter = getHighlighter(wTerms, HighlighterTest.this);
+				highlighter.setTextFragmenter(new SimpleFragmenter(2));
 
-        for (int i = 0; i < hits.length(); i++) {
-          String text = hits.doc(i).get(FIELD_NAME);
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream, HighlighterTest.this, false);
+				result = highlighter.getBestFragment(tokenStream, texts[0])
+						.trim();
+				assertTrue(
+						"Failed to find best section using weighted terms. Found: "
+								+ result, "<B>kennedy</B>".equals(result));
+			}
 
-          highlighter.setTextFragmenter(new SimpleFragmenter(40));
+		};
 
-          String highlightedText = highlighter.getBestFragments(tokenStream, text,
-              maxNumFragmentsRequired, "...");
+		helper.start();
 
-          System.out.println(highlightedText);
-        }
-        // We expect to have zero highlights if the query is multi-terms and is
-        // not
-        // rewritten!
-        assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-            numHighlights == 0);
-      }
-    };
+	}
 
-    helper.start();
-  }
+	// tests a "complex" analyzer that produces multiple
+	// overlapping tokens
+	public void testOverlapAnalyzer() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-  public void testNoFragments() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+			public void run() throws Exception {
+				HashMap synonyms = new HashMap();
+				synonyms.put("football", "soccer,footie");
+				Analyzer analyzer = new SynonymAnalyzer(synonyms);
+				String srchkey = "football";
 
-      public void run() throws Exception {
-        doSearching("AnInvalidQueryWhichShouldYieldNoResults");
+				String s = "football-soccer in the euro 2004 footie competition";
+				QueryParser parser = new QueryParser("bookid", analyzer);
+				Query query = parser.parse(srchkey);
 
-        for (int i = 0; i < texts.length; i++) {
-          String text = texts[i];
-          TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-          Highlighter highlighter = getHighlighter(query, FIELD_NAME, tokenStream,
-              HighlighterTest.this);
-          String result = highlighter.getBestFragment(tokenStream, text);
-          assertNull("The highlight result should be null for text with no query terms", result);
-        }
-      }
-    };
+				TokenStream tokenStream = analyzer.tokenStream(null,
+						new StringReader(s));
 
-    helper.start();
-  }
+				Highlighter highlighter = getHighlighter(query, null,
+						tokenStream, HighlighterTest.this);
 
-  /**
-   * Demonstrates creation of an XHTML compliant doc using new encoding facilities.
-   * 
-   * @throws Exception
-   */
-  public void testEncoding() throws Exception {
+				// Get 3 best fragments and seperate with a "..."
+				tokenStream = analyzer.tokenStream(null, new StringReader(s));
 
-    String rawDocContent = "\"Smith & sons' prices < 3 and >4\" claims article";
-    // run the highlighter on the raw content (scorer does not score any tokens
-    // for
-    // highlighting but scores a single fragment for selection
-    Highlighter highlighter = new Highlighter(this, new SimpleHTMLEncoder(), new Scorer() {
-      public void startFragment(TextFragment newFragment) {
-      }
+				String result = highlighter.getBestFragments(tokenStream, s, 3,
+						"...");
+				String expectedResult = "<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
+				assertTrue(
+						"overlapping analyzer should handle highlights OK, expected:"
+								+ expectedResult + " actual:" + result,
+						expectedResult.equals(result));
+			}
 
-      public float getTokenScore() {
-        return 0;
-      }
+		};
 
-      public float getFragmentScore() {
-        return 1;
-      }
+		helper.start();
 
-      public TokenStream init(TokenStream tokenStream) {
-        return null;
-      }
-    });
-    highlighter.setTextFragmenter(new SimpleFragmenter(2000));
-    TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(rawDocContent));
+	}
 
-    String encodedSnippet = highlighter.getBestFragments(tokenStream, rawDocContent, 1, "");
-    // An ugly bit of XML creation:
-    String xhtml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
-        + "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n"
-        + "<head>\n" + "<title>My Test HTML Document</title>\n" + "</head>\n" + "<body>\n" + "<h2>"
-        + encodedSnippet + "</h2>\n" + "</body>\n" + "</html>";
-    // now an ugly built of XML parsing to test the snippet is encoded OK
-    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
-    DocumentBuilder db = dbf.newDocumentBuilder();
-    org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml.getBytes()));
-    Element root = doc.getDocumentElement();
-    NodeList nodes = root.getElementsByTagName("body");
-    Element body = (Element) nodes.item(0);
-    nodes = body.getElementsByTagName("h2");
-    Element h2 = (Element) nodes.item(0);
-    String decodedSnippet = h2.getFirstChild().getNodeValue();
-    assertEquals("XHTML Encoding should have worked:", rawDocContent, decodedSnippet);
-  }
+	public void testGetSimpleHighlight() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-  public void testMultiSearcher() throws Exception {
-    // setup index 1
-    RAMDirectory ramDir1 = new RAMDirectory();
-    IndexWriter writer1 = new IndexWriter(ramDir1, new StandardAnalyzer(), true);
-    Document d = new Document();
-    Field f = new Field(FIELD_NAME, "multiOne", Field.Store.YES, Field.Index.ANALYZED);
-    d.add(f);
-    writer1.addDocument(d);
-    writer1.optimize();
-    writer1.close();
-    IndexReader reader1 = IndexReader.open(ramDir1);
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("Kennedy");
+				// new Highlighter(HighlighterTest.this, new
+				// QueryTermScorer(query));
 
-    // setup index 2
-    RAMDirectory ramDir2 = new RAMDirectory();
-    IndexWriter writer2 = new IndexWriter(ramDir2, new StandardAnalyzer(), true);
-    d = new Document();
-    f = new Field(FIELD_NAME, "multiTwo", Field.Store.YES, Field.Index.ANALYZED);
-    d.add(f);
-    writer2.addDocument(d);
-    writer2.optimize();
-    writer2.close();
-    IndexReader reader2 = IndexReader.open(ramDir2);
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);
+					String result = highlighter.getBestFragment(tokenStream,
+							text);
+					System.out.println("\t" + result);
+				}
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 4);
+			}
+		};
+		helper.start();
+	}
 
-    IndexSearcher searchers[] = new IndexSearcher[2];
-    searchers[0] = new IndexSearcher(ramDir1);
-    searchers[1] = new IndexSearcher(ramDir2);
-    MultiSearcher multiSearcher = new MultiSearcher(searchers);
-    QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer());
-    parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
-    query = parser.parse("multi*");
-    System.out.println("Searching for: " + query.toString(FIELD_NAME));
-    // at this point the multisearcher calls combine(query[])
-    hits = multiSearcher.search(query);
+	public void testGetTextFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    // query = QueryParser.parse("multi*", FIELD_NAME, new StandardAnalyzer());
-    Query expandedQueries[] = new Query[2];
-    expandedQueries[0] = query.rewrite(reader1);
-    expandedQueries[1] = query.rewrite(reader2);
-    query = query.combine(expandedQueries);
+			public void run() throws Exception {
 
-    // create an instance of the highlighter with the tags used to surround
-    // highlighted text
-    Highlighter highlighter = new Highlighter(this, new QueryTermScorer(query));
+				doSearching("Kennedy");
 
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-      String highlightedText = highlighter.getBestFragment(tokenStream, text);
-      System.out.println(highlightedText);
-    }
-    assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-        numHighlights == 2);
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
 
-  }
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);// new
+					// Highlighter(this,
+					// new
+					// QueryTermScorer(query));
+					highlighter.setTextFragmenter(new SimpleFragmenter(20));
+					String stringResults[] = highlighter.getBestFragments(
+							tokenStream, text, 10);
 
-  public void testFieldSpecificHighlighting() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+					tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					TextFragment fragmentResults[] = highlighter
+							.getBestTextFragments(tokenStream, text, true, 10);
 
-      public void run() throws Exception {
-        String docMainText = "fred is one of the people";
-        QueryParser parser = new QueryParser(FIELD_NAME, analyzer);
-        Query query = parser.parse("fred category:people");
+					assertTrue(
+							"Failed to find correct number of text Fragments: "
+									+ fragmentResults.length + " vs "
+									+ stringResults.length,
+							fragmentResults.length == stringResults.length);
+					for (int j = 0; j < stringResults.length; j++) {
+						System.out.println(fragmentResults[j]);
+						assertTrue("Failed to find same text Fragments: "
+								+ fragmentResults[j] + " found",
+								fragmentResults[j].toString().equals(
+										stringResults[j]));
 
-        // highlighting respects fieldnames used in query
+					}
 
-        Scorer fieldSpecificScorer = null;
-        if (mode == this.QUERY) {
-          fieldSpecificScorer = new QueryScorer(query, FIELD_NAME);
-        } else if (mode == this.QUERY_TERM) {
-          fieldSpecificScorer = new QueryTermScorer(query, "contents");
-        }
-        Highlighter fieldSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
-            fieldSpecificScorer);
-        fieldSpecificHighlighter.setTextFragmenter(new NullFragmenter());
-        String result = fieldSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);
-        assertEquals("Should match", result, "<B>fred</B> is one of the people");
+				}
+			}
+		};
+		helper.start();
+	}
 
-        // highlighting does not respect fieldnames used in query
-        Scorer fieldInSpecificScorer = null;
-        if (mode == this.QUERY) {
-          fieldInSpecificScorer = new QueryScorer(query, null);
-        } else if (mode == this.QUERY_TERM) {
-          fieldInSpecificScorer = new QueryTermScorer(query);
-        }
+	public void testMaxSizeHighlight() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-        Highlighter fieldInSpecificHighlighter = new Highlighter(new SimpleHTMLFormatter(),
-            fieldInSpecificScorer);
-        fieldInSpecificHighlighter.setTextFragmenter(new NullFragmenter());
-        result = fieldInSpecificHighlighter.getBestFragment(analyzer, FIELD_NAME, docMainText);
-        assertEquals("Should match", result, "<B>fred</B> is one of the <B>people</B>");
+			public void run() throws Exception {
+				numHighlights = 0;
+				doSearching("meat");
+				TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+						new StringReader(texts[0]));
+				Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+						tokenStream, HighlighterTest.this);// new
+				// Highlighter(this,
+				// new
+				// QueryTermScorer(query));
+				highlighter.setMaxDocBytesToAnalyze(30);
 
-        reader.close();
-      }
-    };
+				highlighter.getBestFragment(tokenStream, texts[0]);
+				assertTrue(
+						"Setting MaxDocBytesToAnalyze should have prevented "
+								+ "us from finding matches for this record: "
+								+ numHighlights + " found", numHighlights == 0);
+			}
+		};
 
-    helper.start();
+		helper.start();
+	}
 
-  }
+	public void testMaxSizeHighlightTruncates() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-  protected TokenStream getTS2() {
-    // String s = "Hi-Speed10 foo";
-    return new TokenStream() {
-      Iterator iter;
-      List lst;
-      private TermAttribute termAtt;
-      private PositionIncrementAttribute posIncrAtt;
-      private OffsetAttribute offsetAtt;
-      {
-        termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-        posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
-        offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
-        lst = new ArrayList();
-        Token t;
-        t = createToken("hi", 0, 2);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("hispeed", 0, 8);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("speed", 3, 8);
-        t.setPositionIncrement(0);
-        lst.add(t);
-        t = createToken("10", 8, 10);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("foo", 11, 14);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        iter = lst.iterator();
-      }
+			public void run() throws Exception {
+				String goodWord = "goodtoken";
+				Set stopWords = new HashSet(1);
+				stopWords.add("stoppedtoken");
 
-      public boolean incrementToken() throws IOException {
-        if(iter.hasNext()) {
-          Token token = (Token) iter.next();
-          termAtt.setTermBuffer(token.term());
-          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
-          offsetAtt.setOffset(token.startOffset(), token.endOffset());
-          return true;
-        }
-        return false;
-      }
-     
-    };
-  }
+				TermQuery query = new TermQuery(new Term("data", goodWord));
 
-  // same token-stream as above, but the bigger token comes first this time
-  protected TokenStream getTS2a() {
-    // String s = "Hi-Speed10 foo";
-    return new TokenStream() {
-      Iterator iter;
-      List lst;
-      private TermAttribute termAtt;
-      private PositionIncrementAttribute posIncrAtt;
-      private OffsetAttribute offsetAtt;
-      {
-        termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-        posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
-        offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
-        lst = new ArrayList();
-        Token t;
-        t = createToken("hispeed", 0, 8);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("hi", 0, 2);
-        t.setPositionIncrement(0);
-        lst.add(t);
-        t = createToken("speed", 3, 8);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("10", 8, 10);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        t = createToken("foo", 11, 14);
-        t.setPositionIncrement(1);
-        lst.add(t);
-        iter = lst.iterator();
-      }
+				String match = null;
+				StringBuffer sb = new StringBuffer();
+				sb.append(goodWord);
+				for (int i = 0; i < 10000; i++) {
+					sb.append(" ");
+					// only one stopword
+					sb.append(stopWords.iterator().next());
+				}
+				SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
+				Highlighter hg = getHighlighter(query, "data",
+						new StandardAnalyzer(stopWords).tokenStream("data",
+								new StringReader(sb.toString())), fm);// new
+				// Highlighter(fm,
+				// new
+				// QueryTermScorer(query));
+				hg.setTextFragmenter(new NullFragmenter());
+				hg.setMaxDocBytesToAnalyze(100);
+				match = hg.getBestFragment(new StandardAnalyzer(stopWords),
+						"data", sb.toString());
+				assertTrue(
+						"Matched text should be no more than 100 chars in length ",
+						match.length() < hg.getMaxDocBytesToAnalyze());
 
-      public boolean incrementToken() throws IOException {
-        if(iter.hasNext()) {
-          Token token = (Token) iter.next();
-          termAtt.setTermBuffer(token.term());
-          posIncrAtt.setPositionIncrement(token.getPositionIncrement());
-          offsetAtt.setOffset(token.startOffset(), token.endOffset());
-          return true;
-        }
-        return false;
-      }
-    };
-  }
+				// add another tokenized word to the overrall length - but set
+				// way
+				// beyond
+				// the length of text under consideration (after a large slug of
+				// stop
+				// words
+				// + whitespace)
+				sb.append(" ");
+				sb.append(goodWord);
+				match = hg.getBestFragment(new StandardAnalyzer(stopWords),
+						"data", sb.toString());
+				assertTrue(
+						"Matched text should be no more than 100 chars in length ",
+						match.length() < hg.getMaxDocBytesToAnalyze());
+			}
+		};
 
-  public void testOverlapAnalyzer2() throws Exception {
-    TestHighlightRunner helper = new TestHighlightRunner() {
+		helper.start();
 
-      public void run() throws Exception {
-        String s = "Hi-Speed10 foo";
+	}
 
-        Query query;
-        Highlighter highlighter;
-        String result;
+	public void testMaxSizeEndHighlight() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
+			public void run() throws Exception {
+				Set stopWords = new HashSet();
+				stopWords.add("in");
+				stopWords.add("it");
+				TermQuery query = new TermQuery(new Term("text", "searchterm"));
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("foo");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("Hi-Speed10 <B>foo</B>", result);
+				String text = "this is a text with searchterm in it";
+				SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
+				Highlighter hg = getHighlighter(query, "text",
+						new StandardAnalyzer(stopWords).tokenStream("text",
+								new StringReader(text)), fm);
+				hg.setTextFragmenter(new NullFragmenter());
+				hg.setMaxDocCharsToAnalyze(36);
+				String match = hg.getBestFragment(new StandardAnalyzer(
+						stopWords), "text", text);
+				assertTrue(
+						"Matched text should contain remainder of text after highlighted query ",
+						match.endsWith("in it"));
+			}
+		};
+		helper.start();
+	}
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("10");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("Hi-Speed<B>10</B> foo", result);
+	public void testUnRewrittenQuery() throws Exception {
+		final TestHighlightRunner helper = new TestHighlightRunner() {
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("<B>Hi</B>-Speed10 foo", result);
+			public void run() throws Exception {
+				numHighlights = 0;
+				// test to show how rewritten query can still be used
+				searcher = new IndexSearcher(ramDir);
+				Analyzer analyzer = new StandardAnalyzer();
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("speed");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("Hi-<B>Speed</B>10 foo", result);
+				QueryParser parser = new QueryParser(FIELD_NAME, analyzer);
+				Query query = parser.parse("JF? or Kenned*");
+				System.out.println("Searching with primitive query");
+				// forget to set this and...
+				// query=query.rewrite(reader);
+				Hits hits = searcher.search(query);
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hispeed");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("<B>Hi-Speed</B>10 foo", result);
+				// create an instance of the highlighter with the tags used to
+				// surround
+				// highlighted text
+				// QueryHighlightExtractor highlighter = new
+				// QueryHighlightExtractor(this,
+				// query, new StandardAnalyzer());
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi speed");
-        highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2(), s, 3, "...");
-        assertEquals("<B>Hi-Speed</B>10 foo", result);
+				int maxNumFragmentsRequired = 3;
 
-        // ///////////////// same tests, just put the bigger overlapping token
-        // first
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("foo");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("Hi-Speed10 <B>foo</B>", result);
+				for (int i = 0; i < hits.length(); i++) {
+					String text = hits.doc(i).get(FIELD_NAME);
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this, false);
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("10");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("Hi-Speed<B>10</B> foo", result);
+					highlighter.setTextFragmenter(new SimpleFragmenter(40));
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("<B>Hi</B>-Speed10 foo", result);
+					String highlightedText = highlighter.getBestFragments(
+							tokenStream, text, maxNumFragmentsRequired, "...");
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("speed");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("Hi-<B>Speed</B>10 foo", result);
+					System.out.println(highlightedText);
+				}
+				// We expect to have zero highlights if the query is multi-terms
+				// and is
+				// not
+				// rewritten!
+				assertTrue("Failed to find correct number of highlights "
+						+ numHighlights + " found", numHighlights == 0);
+			}
+		};
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hispeed");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("<B>Hi-Speed</B>10 foo", result);
+		helper.start();
+	}
 
-        query = new QueryParser("text", new WhitespaceAnalyzer()).parse("hi speed");
-        highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
-        result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
-        assertEquals("<B>Hi-Speed</B>10 foo", result);
-      }
-    };
+	public void testNoFragments() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
 
-    helper.start();
-  }
-  
-  private Directory dir = new RAMDirectory();
-  private Analyzer a = new WhitespaceAnalyzer();
-  
-  public void testWeightedTermsWithDeletes() throws IOException, ParseException, InvalidTokenOffsetsException {
-    makeIndex();
-    deleteDocument();
-    searchIndex();
-  }
-  
-  private Document doc( String f, String v ){
-    Document doc = new Document();
-    doc.add( new Field( f, v, Store.YES, Index.ANALYZED ) );
-    return doc;
-  }
-  
-  private void makeIndex() throws IOException {
-    IndexWriter writer = new IndexWriter( dir, a, MaxFieldLength.LIMITED );
-    writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
-    writer.addDocument( doc( "t_text1", "more random words for second field del" ) );
-    writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
-    writer.addDocument( doc( "t_text1", "more random words for second field" ) );
-    writer.optimize();
-    writer.close();
-  }
-  
-  private void deleteDocument() throws IOException {
-    IndexWriter writer = new IndexWriter( dir, a, false, MaxFieldLength.LIMITED );
-    writer.deleteDocuments( new Term( "t_text1", "del" ) );
-    // To see negative idf, keep comment the following line
-    //writer.optimize();
-    writer.close();
-  }
-  
-  private void searchIndex() throws IOException, ParseException, InvalidTokenOffsetsException {
-    String q = "t_text1:random";
-    QueryParser parser = new QueryParser( "t_text1", a );
-    Query query = parser.parse( q );
-    IndexSearcher searcher = new IndexSearcher( dir );
-    // This scorer can return negative idf -> null fragment
-    Scorer scorer = new QueryTermScorer( query, searcher.getIndexReader(), "t_text1" );
-    // This scorer doesn't use idf (patch version)
-    //Scorer scorer = new QueryTermScorer( query, "t_text1" );
-    Highlighter h = new Highlighter( scorer );
+			public void run() throws Exception {
+				doSearching("AnInvalidQueryWhichShouldYieldNoResults");
 
-    TopDocs hits = searcher.search(query, null, 10);
-    for( int i = 0; i < hits.totalHits; i++ ){
-      Document doc = searcher.doc( hits.scoreDocs[i].doc );
-      String result = h.getBestFragment( a, "t_text1", doc.get( "t_text1" ));
-      System.out.println("result:" +  result);
-      assertEquals("more <B>random</B> words for second field", result);
-    }
-    searcher.close();
-  }
+				for (int i = 0; i < texts.length; i++) {
+					String text = texts[i];
+					TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+							new StringReader(text));
+					Highlighter highlighter = getHighlighter(query, FIELD_NAME,
+							tokenStream, HighlighterTest.this);
+					String result = highlighter.getBestFragment(tokenStream,
+							text);
+					assertNull(
+							"The highlight result should be null for text with no query terms",
+							result);
+				}
+			}
+		};
 
-  /*
-   * 
-   * public void testBigramAnalyzer() throws IOException, ParseException {
-   * //test to ensure analyzers with none-consecutive start/end offsets //dont
-   * double-highlight text //setup index 1 RAMDirectory ramDir = new
-   * RAMDirectory(); Analyzer bigramAnalyzer=new CJKAnalyzer(); IndexWriter
-   * writer = new IndexWriter(ramDir,bigramAnalyzer , true); Document d = new
-   * Document(); Field f = new Field(FIELD_NAME, "java abc def", true, true,
-   * true); d.add(f); writer.addDocument(d); writer.close(); IndexReader reader =
-   * IndexReader.open(ramDir);
-   * 
-   * IndexSearcher searcher=new IndexSearcher(reader); query =
-   * QueryParser.parse("abc", FIELD_NAME, bigramAnalyzer);
-   * System.out.println("Searching for: " + query.toString(FIELD_NAME)); hits =
-   * searcher.search(query);
-   * 
-   * Highlighter highlighter = new Highlighter(this,new
-   * QueryFragmentScorer(query));
-   * 
-   * for (int i = 0; i < hits.length(); i++) { String text =
-   * hits.doc(i).get(FIELD_NAME); TokenStream
-   * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,new StringReader(text));
-   * String highlightedText = highlighter.getBestFragment(tokenStream,text);
-   * System.out.println(highlightedText); } }
-   */
+		helper.start();
+	}
 
-  public String highlightTerm(String originalText, TokenGroup group) {
-    if (group.getTotalScore() <= 0) {
-      return originalText;
-    }
-    numHighlights++; // update stats used in assertions
-    return "<B>" + originalText + "</B>";
-  }
+	/**
+	 * Demonstrates creation of an XHTML compliant doc using new encoding
+	 * facilities.
+	 * 
+	 * @throws Exception
+	 */
+	public void testEncoding() throws Exception {
 
-  public void doSearching(String queryString) throws Exception {
-    QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer());
-    parser.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
-    query = parser.parse(queryString);
-    doSearching(query);
-  }
+		String rawDocContent = "\"Smith & sons' prices < 3 and >4\" claims article";
+		// run the highlighter on the raw content (scorer does not score any
+		// tokens
+		// for
+		// highlighting but scores a single fragment for selection
+		Highlighter highlighter = new Highlighter(this,
+				new SimpleHTMLEncoder(), new Scorer() {
+					public void startFragment(TextFragment newFragment) {
+					}
 
-  public void doSearching(Query unReWrittenQuery) throws Exception {
-    searcher = new IndexSearcher(ramDir);
-    // for any multi-term queries to work (prefix, wildcard, range,fuzzy etc)
-    // you must use a rewritten query!
-    query = unReWrittenQuery.rewrite(reader);
-    System.out.println("Searching for: " + query.toString(FIELD_NAME));
-    hits = searcher.search(query);
-  }
+					public float getTokenScore() {
+						return 0;
+					}
 
-  public void assertExpectedHighlightCount(final int maxNumFragmentsRequired,
-      final int expectedHighlights) throws Exception {
-    for (int i = 0; i < hits.length(); i++) {
-      String text = hits.doc(i).get(FIELD_NAME);
-      TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, new StringReader(text));
-      QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
-      Highlighter highlighter = new Highlighter(this, scorer);
+					public float getFragmentScore() {
+						return 1;
+					}
 
-      highlighter.setTextFragmenter(new SimpleFragmenter(40));
+					public TokenStream init(TokenStream tokenStream) {
+						return null;
+					}
+				});
+		highlighter.setTextFragmenter(new SimpleFragmenter(2000));
+		TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+				new StringReader(rawDocContent));
 
-      String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-          "...");
-      System.out.println("\t" + result);
+		String encodedSnippet = highlighter.getBestFragments(tokenStream,
+				rawDocContent, 1, "");
+		// An ugly bit of XML creation:
+		String xhtml = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+				+ "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\" lang=\"en\">\n"
+				+ "<head>\n" + "<title>My Test HTML Document</title>\n"
+				+ "</head>\n" + "<body>\n" + "<h2>" + encodedSnippet
+				+ "</h2>\n" + "</body>\n" + "</html>";
+		// now an ugly built of XML parsing to test the snippet is encoded OK
+		DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+		DocumentBuilder db = dbf.newDocumentBuilder();
+		org.w3c.dom.Document doc = db.parse(new ByteArrayInputStream(xhtml
+				.getBytes()));
+		Element root = doc.getDocumentElement();
+		NodeList nodes = root.getElementsByTagName("body");
+		Element body = (Element) nodes.item(0);
+		nodes = body.getElementsByTagName("h2");
+		Element h2 = (Element) nodes.item(0);
+		String decodedSnippet = h2.getFirstChild().getNodeValue();
+		assertEquals("XHTML Encoding should have worked:", rawDocContent,
+				decodedSnippet);
+	}
 
-      assertTrue("Failed to find correct number of highlights " + numHighlights + " found",
-          numHighlights == expectedHighlights);
-    }
-  }
+	public void testMultiSearcher() throws Exception {
+		// setup index 1
+		RAMDirectory ramDir1 = new RAMDirectory();
+		IndexWriter writer1 = new IndexWriter(ramDir1, new StandardAnalyzer(),
+				true);
+		Document d = new Document();
+		Field f = new Field(FIELD_NAME, "multiOne", Field.Store.YES,
+				Field.Index.ANALYZED);
+		d.add(f);
+		writer1.addDocument(d);
+		writer1.optimize();
+		writer1.close();
+		IndexReader reader1 = IndexReader.open(ramDir1);
 
-  protected void setUp() throws Exception {
-    super.setUp();
-    ramDir = new RAMDirectory();
-    IndexWriter writer = new IndexWriter(ramDir, new StandardAnalyzer(), true);
-    for (int i = 0; i < texts.length; i++) {
-      addDoc(writer, texts[i]);
-    }
-    Document doc = new Document();
-    NumericField nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
-    nfield.setIntValue(1);
-    doc.add(nfield);
-    writer.addDocument(doc, analyzer);
-    nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
-    nfield.setIntValue(3);
-    doc = new Document();
-    doc.add(nfield);
-    writer.addDocument(doc, analyzer);
-    nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
-    nfield.setIntValue(5);
-    doc = new Document();
-    doc.add(nfield);
-    writer.addDocument(doc, analyzer);
-    nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
-    nfield.setIntValue(7);
-    doc = new Document();
-    doc.add(nfield);
-    writer.addDocument(doc, analyzer);
-    writer.optimize();
-    writer.close();
-    reader = IndexReader.open(ramDir);
-    numHighlights = 0;
-  }
+		// setup index 2
+		RAMDirectory ramDir2 = new RAMDirectory();
+		IndexWriter writer2 = new IndexWriter(ramDir2, new StandardAnalyzer(),
+				true);
+		d = new Document();
+		f = new Field(FIELD_NAME, "multiTwo", Field.Store.YES,
+				Field.Index.ANALYZED);
+		d.add(f);
+		writer2.addDocument(d);
+		writer2.optimize();
+		writer2.close();
+		IndexReader reader2 = IndexReader.open(ramDir2);
 
-  private void addDoc(IndexWriter writer, String text) throws IOException {
-    Document d = new Document();
-    Field f = new Field(FIELD_NAME, text, Field.Store.YES, Field.Index.ANALYZED);
-    d.add(f);
-    writer.addDocument(d);
+		IndexSearcher searchers[] = new IndexSearcher[2];
+		searchers[0] = new IndexSearcher(ramDir1);
+		searchers[1] = new IndexSearcher(ramDir2);
+		MultiSearcher multiSearcher = new MultiSearcher(searchers);
+		QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer());
+		parser
+				.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+		query = parser.parse("multi*");
+		System.out.println("Searching for: " + query.toString(FIELD_NAME));
+		// at this point the multisearcher calls combine(query[])
+		hits = multiSearcher.search(query);
 
-  }
+		// query = QueryParser.parse("multi*", FIELD_NAME, new
+		// StandardAnalyzer());
+		Query expandedQueries[] = new Query[2];
+		expandedQueries[0] = query.rewrite(reader1);
+		expandedQueries[1] = query.rewrite(reader2);
+		query = query.combine(expandedQueries);
 
-  protected void tearDown() throws Exception {
-    super.tearDown();
-  }
+		// create an instance of the highlighter with the tags used to surround
+		// highlighted text
+		Highlighter highlighter = new Highlighter(this, new QueryTermScorer(
+				query));
 
-  private static Token createToken(String term, int start, int offset)
-  {
-    Token token = new Token(start, offset);
-    token.setTermBuffer(term);
-    return token;
-  }
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
+			String highlightedText = highlighter.getBestFragment(tokenStream,
+					text);
+			System.out.println(highlightedText);
+		}
+		assertTrue("Failed to find correct number of highlights "
+				+ numHighlights + " found", numHighlights == 2);
 
+	}
+
+	public void testFieldSpecificHighlighting() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
+
+			public void run() throws Exception {
+				String docMainText = "fred is one of the people";
+				QueryParser parser = new QueryParser(FIELD_NAME, analyzer);
+				Query query = parser.parse("fred category:people");
+
+				// highlighting respects fieldnames used in query
+
+				Scorer fieldSpecificScorer = null;
+				if (mode == this.QUERY) {
+					fieldSpecificScorer = new QueryScorer(query, FIELD_NAME);
+				} else if (mode == this.QUERY_TERM) {
+					fieldSpecificScorer = new QueryTermScorer(query, "contents");
+				}
+				Highlighter fieldSpecificHighlighter = new Highlighter(
+						new SimpleHTMLFormatter(), fieldSpecificScorer);
+				fieldSpecificHighlighter
+						.setTextFragmenter(new NullFragmenter());
+				String result = fieldSpecificHighlighter.getBestFragment(
+						analyzer, FIELD_NAME, docMainText);
+				assertEquals("Should match", result,
+						"<B>fred</B> is one of the people");
+
+				// highlighting does not respect fieldnames used in query
+				Scorer fieldInSpecificScorer = null;
+				if (mode == this.QUERY) {
+					fieldInSpecificScorer = new QueryScorer(query, null);
+				} else if (mode == this.QUERY_TERM) {
+					fieldInSpecificScorer = new QueryTermScorer(query);
+				}
+
+				Highlighter fieldInSpecificHighlighter = new Highlighter(
+						new SimpleHTMLFormatter(), fieldInSpecificScorer);
+				fieldInSpecificHighlighter
+						.setTextFragmenter(new NullFragmenter());
+				result = fieldInSpecificHighlighter.getBestFragment(analyzer,
+						FIELD_NAME, docMainText);
+				assertEquals("Should match", result,
+						"<B>fred</B> is one of the <B>people</B>");
+
+				reader.close();
+			}
+		};
+
+		helper.start();
+
+	}
+
+	protected TokenStream getTS2() {
+		// String s = "Hi-Speed10 foo";
+		return new TokenStream() {
+			Iterator iter;
+			List lst;
+			private TermAttribute termAtt;
+			private PositionIncrementAttribute posIncrAtt;
+			private OffsetAttribute offsetAtt;
+			{
+				termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+				posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+				offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+				lst = new ArrayList();
+				Token t;
+				t = createToken("hi", 0, 2);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("hispeed", 0, 8);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("speed", 3, 8);
+				t.setPositionIncrement(0);
+				lst.add(t);
+				t = createToken("10", 8, 10);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("foo", 11, 14);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				iter = lst.iterator();
+			}
+
+			public boolean incrementToken() throws IOException {
+				if (iter.hasNext()) {
+					Token token = (Token) iter.next();
+					termAtt.setTermBuffer(token.term());
+					posIncrAtt.setPositionIncrement(token
+							.getPositionIncrement());
+					offsetAtt.setOffset(token.startOffset(), token.endOffset());
+					return true;
+				}
+				return false;
+			}
+
+		};
+	}
+
+	// same token-stream as above, but the bigger token comes first this time
+	protected TokenStream getTS2a() {
+		// String s = "Hi-Speed10 foo";
+		return new TokenStream() {
+			Iterator iter;
+			List lst;
+			private TermAttribute termAtt;
+			private PositionIncrementAttribute posIncrAtt;
+			private OffsetAttribute offsetAtt;
+			{
+				termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+				posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+				offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+				lst = new ArrayList();
+				Token t;
+				t = createToken("hispeed", 0, 8);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("hi", 0, 2);
+				t.setPositionIncrement(0);
+				lst.add(t);
+				t = createToken("speed", 3, 8);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("10", 8, 10);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				t = createToken("foo", 11, 14);
+				t.setPositionIncrement(1);
+				lst.add(t);
+				iter = lst.iterator();
+			}
+
+			public boolean incrementToken() throws IOException {
+				if (iter.hasNext()) {
+					Token token = (Token) iter.next();
+					termAtt.setTermBuffer(token.term());
+					posIncrAtt.setPositionIncrement(token
+							.getPositionIncrement());
+					offsetAtt.setOffset(token.startOffset(), token.endOffset());
+					return true;
+				}
+				return false;
+			}
+		};
+	}
+
+	public void testOverlapAnalyzer2() throws Exception {
+		TestHighlightRunner helper = new TestHighlightRunner() {
+
+			public void run() throws Exception {
+				String s = "Hi-Speed10 foo";
+
+				Query query;
+				Highlighter highlighter;
+				String result;
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("foo");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("Hi-Speed10 <B>foo</B>", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("10");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("Hi-Speed<B>10</B> foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hi");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("<B>Hi</B>-Speed10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("speed");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("Hi-<B>Speed</B>10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hispeed");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("<B>Hi-Speed</B>10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hi speed");
+				highlighter = getHighlighter(query, "text", getTS2(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2(), s, 3, "...");
+				assertEquals("<B>Hi-Speed</B>10 foo", result);
+
+				// ///////////////// same tests, just put the bigger overlapping
+				// token
+				// first
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("foo");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("Hi-Speed10 <B>foo</B>", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("10");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("Hi-Speed<B>10</B> foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hi");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("<B>Hi</B>-Speed10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("speed");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("Hi-<B>Speed</B>10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hispeed");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("<B>Hi-Speed</B>10 foo", result);
+
+				query = new QueryParser("text", new WhitespaceAnalyzer())
+						.parse("hi speed");
+				highlighter = getHighlighter(query, "text", getTS2a(),
+						HighlighterTest.this);
+				result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
+				assertEquals("<B>Hi-Speed</B>10 foo", result);
+			}
+		};
+
+		helper.start();
+	}
+
+	private Directory dir = new RAMDirectory();
+	private Analyzer a = new WhitespaceAnalyzer();
+
+	public void testWeightedTermsWithDeletes() throws IOException,
+			ParseException, InvalidTokenOffsetsException {
+		makeIndex();
+		deleteDocument();
+		searchIndex();
+	}
+
+	private Document doc(String f, String v) {
+		Document doc = new Document();
+		doc.add(new Field(f, v, Store.YES, Index.ANALYZED));
+		return doc;
+	}
+
+	private void makeIndex() throws IOException {
+		IndexWriter writer = new IndexWriter(dir, a, MaxFieldLength.LIMITED);
+		writer.addDocument(doc("t_text1",
+				"random words for highlighting tests del"));
+		writer.addDocument(doc("t_text1",
+				"more random words for second field del"));
+		writer.addDocument(doc("t_text1",
+				"random words for highlighting tests del"));
+		writer
+				.addDocument(doc("t_text1",
+						"more random words for second field"));
+		writer.optimize();
+		writer.close();
+	}
+
+	private void deleteDocument() throws IOException {
+		IndexWriter writer = new IndexWriter(dir, a, false,
+				MaxFieldLength.LIMITED);
+		writer.deleteDocuments(new Term("t_text1", "del"));
+		// To see negative idf, keep comment the following line
+		// writer.optimize();
+		writer.close();
+	}
+
+	private void searchIndex() throws IOException, ParseException,
+			InvalidTokenOffsetsException {
+		String q = "t_text1:random";
+		QueryParser parser = new QueryParser("t_text1", a);
+		Query query = parser.parse(q);
+		IndexSearcher searcher = new IndexSearcher(dir);
+		// This scorer can return negative idf -> null fragment
+		Scorer scorer = new QueryTermScorer(query, searcher.getIndexReader(),
+				"t_text1");
+		// This scorer doesn't use idf (patch version)
+		// Scorer scorer = new QueryTermScorer( query, "t_text1" );
+		Highlighter h = new Highlighter(scorer);
+
+		TopDocs hits = searcher.search(query, null, 10);
+		for (int i = 0; i < hits.totalHits; i++) {
+			Document doc = searcher.doc(hits.scoreDocs[i].doc);
+			String result = h.getBestFragment(a, "t_text1", doc.get("t_text1"));
+			System.out.println("result:" + result);
+			assertEquals("more <B>random</B> words for second field", result);
+		}
+		searcher.close();
+	}
+
+	/*
+	 * 
+	 * public void testBigramAnalyzer() throws IOException, ParseException {
+	 * //test to ensure analyzers with none-consecutive start/end offsets //dont
+	 * double-highlight text //setup index 1 RAMDirectory ramDir = new
+	 * RAMDirectory(); Analyzer bigramAnalyzer=new CJKAnalyzer(); IndexWriter
+	 * writer = new IndexWriter(ramDir,bigramAnalyzer , true); Document d = new
+	 * Document(); Field f = new Field(FIELD_NAME, "java abc def", true, true,
+	 * true); d.add(f); writer.addDocument(d); writer.close(); IndexReader
+	 * reader = IndexReader.open(ramDir);
+	 * 
+	 * IndexSearcher searcher=new IndexSearcher(reader); query =
+	 * QueryParser.parse("abc", FIELD_NAME, bigramAnalyzer);
+	 * System.out.println("Searching for: " + query.toString(FIELD_NAME)); hits
+	 * = searcher.search(query);
+	 * 
+	 * Highlighter highlighter = new Highlighter(this,new
+	 * QueryFragmentScorer(query));
+	 * 
+	 * for (int i = 0; i < hits.length(); i++) { String text =
+	 * hits.doc(i).get(FIELD_NAME); TokenStream
+	 * tokenStream=bigramAnalyzer.tokenStream(FIELD_NAME,new
+	 * StringReader(text)); String highlightedText =
+	 * highlighter.getBestFragment(tokenStream,text);
+	 * System.out.println(highlightedText); } }
+	 */
+
+	public String highlightTerm(String originalText, TokenGroup group) {
+		if (group.getTotalScore() <= 0) {
+			return originalText;
+		}
+		numHighlights++; // update stats used in assertions
+		return "<B>" + originalText + "</B>";
+	}
+
+	public void doSearching(String queryString) throws Exception {
+		QueryParser parser = new QueryParser(FIELD_NAME, new StandardAnalyzer());
+		parser
+				.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+		query = parser.parse(queryString);
+		doSearching(query);
+	}
+
+	public void doSearching(Query unReWrittenQuery) throws Exception {
+		searcher = new IndexSearcher(ramDir);
+		// for any multi-term queries to work (prefix, wildcard, range,fuzzy
+		// etc)
+		// you must use a rewritten query!
+		query = unReWrittenQuery.rewrite(reader);
+		System.out.println("Searching for: " + query.toString(FIELD_NAME));
+		hits = searcher.search(query);
+	}
+
+	public void assertExpectedHighlightCount(final int maxNumFragmentsRequired,
+			final int expectedHighlights) throws Exception {
+		for (int i = 0; i < hits.length(); i++) {
+			String text = hits.doc(i).get(FIELD_NAME);
+			TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME,
+					new StringReader(text));
+			QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
+			Highlighter highlighter = new Highlighter(this, scorer);
+
+			highlighter.setTextFragmenter(new SimpleFragmenter(40));
+
+			String result = highlighter.getBestFragments(tokenStream, text,
+					maxNumFragmentsRequired, "...");
+			System.out.println("\t" + result);
+
+			assertTrue("Failed to find correct number of highlights "
+					+ numHighlights + " found",
+					numHighlights == expectedHighlights);
+		}
+	}
+
+	protected void setUp() throws Exception {
+		super.setUp();
+		ramDir = new RAMDirectory();
+		IndexWriter writer = new IndexWriter(ramDir, new StandardAnalyzer(),
+				true);
+		for (int i = 0; i < texts.length; i++) {
+			addDoc(writer, texts[i]);
+		}
+		Document doc = new Document();
+		NumericField nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES,
+				true);
+		nfield.setIntValue(1);
+		doc.add(nfield);
+		writer.addDocument(doc, analyzer);
+		nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+		nfield.setIntValue(3);
+		doc = new Document();
+		doc.add(nfield);
+		writer.addDocument(doc, analyzer);
+		nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+		nfield.setIntValue(5);
+		doc = new Document();
+		doc.add(nfield);
+		writer.addDocument(doc, analyzer);
+		nfield = new NumericField(NUMERIC_FIELD_NAME, Store.YES, true);
+		nfield.setIntValue(7);
+		doc = new Document();
+		doc.add(nfield);
+		writer.addDocument(doc, analyzer);
+		writer.optimize();
+		writer.close();
+		reader = IndexReader.open(ramDir);
+		numHighlights = 0;
+	}
+
+	private void addDoc(IndexWriter writer, String text) throws IOException {
+		Document d = new Document();
+		Field f = new Field(FIELD_NAME, text, Field.Store.YES,
+				Field.Index.ANALYZED);
+		d.add(f);
+		writer.addDocument(d);
+
+	}
+
+	protected void tearDown() throws Exception {
+		super.tearDown();
+	}
+
+	private static Token createToken(String term, int start, int offset) {
+		Token token = new Token(start, offset);
+		token.setTermBuffer(term);
+		return token;
+	}
+
 }
 
 // ===================================================================
@@ -1755,167 +2022,182 @@
 // ===================================================================
 
 class SynonymAnalyzer extends Analyzer {
-  private Map synonyms;
+	private Map synonyms;
 
-  public SynonymAnalyzer(Map synonyms) {
-    this.synonyms = synonyms;
-  }
+	public SynonymAnalyzer(Map synonyms) {
+		this.synonyms = synonyms;
+	}
 
-  /*
-   * (non-Javadoc)
-   * 
-   * @see org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String,
-   *      java.io.Reader)
-   */
-  public TokenStream tokenStream(String arg0, Reader arg1) {
-    LowerCaseTokenizer stream = new LowerCaseTokenizer(arg1);
-    stream.addAttribute(TermAttribute.class);
-    stream.addAttribute(PositionIncrementAttribute.class);
-    stream.addAttribute(OffsetAttribute.class);
-    return new SynonymTokenizer(stream, synonyms);
-  }
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.lucene.analysis.Analyzer#tokenStream(java.lang.String,
+	 * java.io.Reader)
+	 */
+	public TokenStream tokenStream(String arg0, Reader arg1) {
+		LowerCaseTokenizer stream = new LowerCaseTokenizer(arg1);
+		stream.addAttribute(TermAttribute.class);
+		stream.addAttribute(PositionIncrementAttribute.class);
+		stream.addAttribute(OffsetAttribute.class);
+		return new SynonymTokenizer(stream, synonyms);
+	}
 }
 
 /**
- * Expands a token stream with synonyms (TODO - make the synonyms analyzed by choice of analyzer)
- *
+ * Expands a token stream with synonyms (TODO - make the synonyms analyzed by
+ * choice of analyzer)
+ * 
  */
 class SynonymTokenizer extends TokenStream {
-  private TokenStream realStream;
-  private Token currentRealToken = null;
-  private org.apache.lucene.analysis.Token cRealToken = null;
-  private Map synonyms;
-  StringTokenizer st = null;
-  private TermAttribute realTermAtt;
-  private PositionIncrementAttribute realPosIncrAtt;
-  private OffsetAttribute realOffsetAtt;
-  private TermAttribute termAtt;
-  private PositionIncrementAttribute posIncrAtt;
-  private OffsetAttribute offsetAtt;
+	private TokenStream realStream;
+	private Token currentRealToken = null;
+	private org.apache.lucene.analysis.Token cRealToken = null;
+	private Map synonyms;
+	StringTokenizer st = null;
+	private TermAttribute realTermAtt;
+	private PositionIncrementAttribute realPosIncrAtt;
+	private OffsetAttribute realOffsetAtt;
+	private TermAttribute termAtt;
+	private PositionIncrementAttribute posIncrAtt;
+	private OffsetAttribute offsetAtt;
 
-  public SynonymTokenizer(TokenStream realStream, Map synonyms) {
-    this.realStream = realStream;
-    this.synonyms = synonyms;
-    realTermAtt = (TermAttribute) realStream.addAttribute(TermAttribute.class);
-    realPosIncrAtt = (PositionIncrementAttribute) realStream.addAttribute(PositionIncrementAttribute.class);
-    realOffsetAtt = (OffsetAttribute) realStream.addAttribute(OffsetAttribute.class);
+	public SynonymTokenizer(TokenStream realStream, Map synonyms) {
+		this.realStream = realStream;
+		this.synonyms = synonyms;
+		realTermAtt = (TermAttribute) realStream
+				.addAttribute(TermAttribute.class);
+		realPosIncrAtt = (PositionIncrementAttribute) realStream
+				.addAttribute(PositionIncrementAttribute.class);
+		realOffsetAtt = (OffsetAttribute) realStream
+				.addAttribute(OffsetAttribute.class);
 
-    termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-    posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
-    offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
-  }
+		termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+		posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+		offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+	}
 
-  public boolean incrementToken() throws IOException {
+	public boolean incrementToken() throws IOException {
 
-    if (currentRealToken == null) {
-      boolean next = realStream.incrementToken();
-      if (!next) {
-        return false;
-      }
-      //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
-      termAtt.setTermBuffer(realTermAtt.term());
-      offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
-      posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
+		if (currentRealToken == null) {
+			boolean next = realStream.incrementToken();
+			if (!next) {
+				return false;
+			}
+			// Token nextRealToken = new Token(, offsetAtt.startOffset(),
+			// offsetAtt.endOffset());
+			termAtt.setTermBuffer(realTermAtt.term());
+			offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt
+					.endOffset());
+			posIncrAtt.setPositionIncrement(realPosIncrAtt
+					.getPositionIncrement());
 
-      String expansions = (String) synonyms.get(realTermAtt.term());
-      if (expansions == null) {
-        return true;
-      }
-      st = new StringTokenizer(expansions, ",");
-      if (st.hasMoreTokens()) {
-        currentRealToken = new Token(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
-        currentRealToken.setTermBuffer(realTermAtt.term());
-      }
-      
-      return true;
-    } else {
-      String tok = st.nextToken();
-      termAtt.setTermBuffer(tok);
-      offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
-      posIncrAtt.setPositionIncrement(0);
-      if (!st.hasMoreTokens()) {
-        currentRealToken = null;
-        st = null;
-      }
-      return true;
-    }
-    
-  }
+			String expansions = (String) synonyms.get(realTermAtt.term());
+			if (expansions == null) {
+				return true;
+			}
+			st = new StringTokenizer(expansions, ",");
+			if (st.hasMoreTokens()) {
+				currentRealToken = new Token(realOffsetAtt.startOffset(),
+						realOffsetAtt.endOffset());
+				currentRealToken.setTermBuffer(realTermAtt.term());
+			}
 
-  static abstract class TestHighlightRunner {
-    static final int QUERY = 0;
-    static final int QUERY_TERM = 1;
+			return true;
+		} else {
+			String tok = st.nextToken();
+			termAtt.setTermBuffer(tok);
+			offsetAtt.setOffset(currentRealToken.startOffset(),
+					currentRealToken.endOffset());
+			posIncrAtt.setPositionIncrement(0);
+			if (!st.hasMoreTokens()) {
+				currentRealToken = null;
+				st = null;
+			}
+			return true;
+		}
 
-    int mode = QUERY;
-    Fragmenter frag = new SimpleFragmenter(20);
-    
-    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream, Formatter formatter) {
-      return getHighlighter(query, fieldName, stream, formatter, true);
-    }
-    
-    public Highlighter getHighlighter(Query query, String fieldName, TokenStream stream, Formatter formatter, boolean expanMultiTerm) {
-      Scorer scorer = null;
-      if (mode == QUERY) {
-        scorer = new QueryScorer(query, fieldName);
-        if(!expanMultiTerm) {
-          ((QueryScorer)scorer).setExpandMultiTermQuery(false);
-        }
-      } else if (mode == QUERY_TERM) {
-        scorer = new QueryTermScorer(query);
-      } else {
-        throw new RuntimeException("Unknown highlight mode");
-      }
-      
-      return new Highlighter(formatter, scorer);
-    }
+	}
 
-    Highlighter getHighlighter(WeightedTerm[] weightedTerms, Formatter formatter) {
-      if (mode == QUERY) {
-        return  new Highlighter(formatter, new QueryScorer((WeightedSpanTerm[]) weightedTerms));
-      } else if (mode == QUERY_TERM) {
-        return new Highlighter(formatter, new QueryTermScorer(weightedTerms));
+	static abstract class TestHighlightRunner {
+		static final int QUERY = 0;
+		static final int QUERY_TERM = 1;
 
-      } else {
-        throw new RuntimeException("Unknown highlight mode");
-      }
-    }
+		int mode = QUERY;
+		Fragmenter frag = new SimpleFragmenter(20);
 
-    void doStandardHighlights(Analyzer analyzer, Hits hits, Query query, Formatter formatter)
-    throws Exception {
-      doStandardHighlights(analyzer, hits, query, formatter, false);
-    }
-    
-    void doStandardHighlights(Analyzer analyzer, Hits hits, Query query, Formatter formatter, boolean expandMT)
-        throws Exception {
+		public Highlighter getHighlighter(Query query, String fieldName,
+				TokenStream stream, Formatter formatter) {
+			return getHighlighter(query, fieldName, stream, formatter, true);
+		}
 
-      for (int i = 0; i < hits.length(); i++) {
-        String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
-        int maxNumFragmentsRequired = 2;
-        String fragmentSeparator = "...";
-        Scorer scorer = null;
-        TokenStream tokenStream = analyzer.tokenStream(HighlighterTest.FIELD_NAME, new StringReader(text));
-        if (mode == QUERY) {
-          scorer = new QueryScorer(query);
-        } else if (mode == QUERY_TERM) {
-          scorer = new QueryTermScorer(query);
-        }
-        Highlighter highlighter = new Highlighter(formatter, scorer);
-        highlighter.setTextFragmenter(frag);
+		public Highlighter getHighlighter(Query query, String fieldName,
+				TokenStream stream, Formatter formatter, boolean expanMultiTerm) {
+			Scorer scorer = null;
+			if (mode == QUERY) {
+				scorer = new QueryScorer(query, fieldName);
+				if (!expanMultiTerm) {
+					((QueryScorer) scorer).setExpandMultiTermQuery(false);
+				}
+			} else if (mode == QUERY_TERM) {
+				scorer = new QueryTermScorer(query);
+			} else {
+				throw new RuntimeException("Unknown highlight mode");
+			}
 
-        String result = highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired,
-            fragmentSeparator);
-        System.out.println("\t" + result);
-      }
-    }
+			return new Highlighter(formatter, scorer);
+		}
 
-    abstract void run() throws Exception;
+		Highlighter getHighlighter(WeightedTerm[] weightedTerms,
+				Formatter formatter) {
+			if (mode == QUERY) {
+				return new Highlighter(formatter, new QueryScorer(
+						(WeightedSpanTerm[]) weightedTerms));
+			} else if (mode == QUERY_TERM) {
+				return new Highlighter(formatter, new QueryTermScorer(
+						weightedTerms));
 
-    void start() throws Exception {
-      System.out.println("Run QueryScorer");
-      run();
-      System.out.println("Run QueryTermScorer");
-      mode = QUERY_TERM;
-      run();
-    }
-  }
+			} else {
+				throw new RuntimeException("Unknown highlight mode");
+			}
+		}
+
+		void doStandardHighlights(Analyzer analyzer, Hits hits, Query query,
+				Formatter formatter) throws Exception {
+			doStandardHighlights(analyzer, hits, query, formatter, false);
+		}
+
+		void doStandardHighlights(Analyzer analyzer, Hits hits, Query query,
+				Formatter formatter, boolean expandMT) throws Exception {
+
+			for (int i = 0; i < hits.length(); i++) {
+				String text = hits.doc(i).get(HighlighterTest.FIELD_NAME);
+				int maxNumFragmentsRequired = 2;
+				String fragmentSeparator = "...";
+				Scorer scorer = null;
+				TokenStream tokenStream = analyzer.tokenStream(
+						HighlighterTest.FIELD_NAME, new StringReader(text));
+				if (mode == QUERY) {
+					scorer = new QueryScorer(query);
+				} else if (mode == QUERY_TERM) {
+					scorer = new QueryTermScorer(query);
+				}
+				Highlighter highlighter = new Highlighter(formatter, scorer);
+				highlighter.setTextFragmenter(frag);
+
+				String result = highlighter.getBestFragments(tokenStream, text,
+						maxNumFragmentsRequired, fragmentSeparator);
+				System.out.println("\t" + result);
+			}
+		}
+
+		abstract void run() throws Exception;
+
+		void start() throws Exception {
+			System.out.println("Run QueryScorer");
+			run();
+			System.out.println("Run QueryTermScorer");
+			mode = QUERY_TERM;
+			run();
+		}
+	}
 }
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java	(revision 908726)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java	(working copy)
@@ -28,8 +28,11 @@
 
 import org.apache.lucene.analysis.CachingTokenFilter;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.document.Document;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermPositions;
 import org.apache.lucene.index.memory.MemoryIndex;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
@@ -47,6 +50,8 @@
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.search.WildcardQuery;
 import org.apache.lucene.search.spans.FieldMaskingSpanQuery;
+import org.apache.lucene.search.spans.NearSpansOrdered;
+import org.apache.lucene.search.spans.NearSpansUnordered;
 import org.apache.lucene.search.spans.SpanFirstQuery;
 import org.apache.lucene.search.spans.SpanNearQuery;
 import org.apache.lucene.search.spans.SpanNotQuery;
@@ -54,11 +59,13 @@
 import org.apache.lucene.search.spans.SpanQuery;
 import org.apache.lucene.search.spans.SpanTermQuery;
 import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.search.spans.TermSpans;
 import org.apache.lucene.util.StringHelper;
 
 /**
- * Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether 
- * {@link Term}s from the {@link Query} are contained in a supplied {@link TokenStream}.
+ * Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on
+ * whether {@link Term}s from the {@link Query} are contained in a supplied
+ * {@link TokenStream}.
  */
 public class WeightedSpanTermExtractor {
 
@@ -94,7 +101,8 @@
   }
 
   /**
-   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms from the supplied <code>Query</code>.
+   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms
+   * from the supplied <code>Query</code>.
    * 
    * @param query
    *          Query to extract Terms from
@@ -133,7 +141,7 @@
           }
           lastPos = pos;
         }
-        if(largestInc > 1) {
+        if (largestInc > 1) {
           slop += largestInc;
         }
       }
@@ -154,31 +162,32 @@
     } else if (query instanceof FilteredQuery) {
       extract(((FilteredQuery) query).getQuery(), terms);
     } else if (query instanceof DisjunctionMaxQuery) {
-      for (Iterator iterator = ((DisjunctionMaxQuery) query).iterator(); iterator.hasNext();) {
+      for (Iterator iterator = ((DisjunctionMaxQuery) query).iterator(); iterator
+          .hasNext();) {
         extract((Query) iterator.next(), terms);
       }
     } else if (query instanceof MultiTermQuery && expandMultiTermQuery) {
-      MultiTermQuery mtq = ((MultiTermQuery)query);
-      if(mtq.getRewriteMethod() != MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE) {
+      MultiTermQuery mtq = ((MultiTermQuery) query);
+      if (mtq.getRewriteMethod() != MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE) {
         mtq = copyMultiTermQuery(mtq);
         mtq.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
         query = mtq;
       }
       String field = null;
-      if(mtq instanceof TermRangeQuery) {
-        field = ((TermRangeQuery)mtq).getField();
+      if (mtq instanceof TermRangeQuery) {
+        field = ((TermRangeQuery) mtq).getField();
       } else {
         Term term = mtq.getTerm();
-        if(term != null) {
+        if (term != null) {
           field = term.field();
         }
       }
-      if(field != null) {
+      if (field != null) {
         IndexReader ir = getReaderForField(field);
         extract(query.rewrite(ir), terms);
       }
     } else if (query instanceof RangeQuery) {
-      String field = ((RangeQuery)query).getField();
+      String field = ((RangeQuery) query).getField();
       IndexReader ir = getReaderForField(field);
       extract(query.rewrite(ir), terms);
     } else if (query instanceof MultiPhraseQuery) {
@@ -201,7 +210,8 @@
           final Term[] termArray = (Term[]) termArrays.get(i);
           List disjuncts = disjunctLists[positions[i]];
           if (disjuncts == null) {
-            disjuncts = (disjunctLists[positions[i]] = new ArrayList(termArray.length));
+            disjuncts = (disjunctLists[positions[i]] = new ArrayList(
+                termArray.length));
             ++distinctPositions;
           }
           for (int j = 0; j < termArray.length; ++j) {
@@ -225,7 +235,8 @@
         final int slop = mpq.getSlop();
         final boolean inorder = (slop == 0);
 
-        SpanNearQuery sp = new SpanNearQuery(clauses, slop + positionGaps, inorder);
+        SpanNearQuery sp = new SpanNearQuery(clauses, slop + positionGaps,
+            inorder);
         sp.setBoost(query.getBoost());
         extractWeightedSpanTerms(terms, sp);
       }
@@ -233,7 +244,8 @@
   }
 
   /**
-   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms from the supplied <code>SpanQuery</code>.
+   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms
+   * from the supplied <code>SpanQuery</code>.
    * 
    * @param terms
    *          Map to place created WeightedSpanTerms in
@@ -241,9 +253,12 @@
    *          SpanQuery to extract Terms from
    * @throws IOException
    */
-  private void extractWeightedSpanTerms(Map terms, SpanQuery spanQuery) throws IOException {
+  private void extractWeightedSpanTerms(
+      Map/* <String, WeightedSpanTerm> */terms, SpanQuery spanQuery)
+      throws IOException {
+
     Set fieldNames;
-    
+
     if (fieldName == null) {
       fieldNames = new HashSet();
       collectSpanQueryFields(spanQuery, fieldNames);
@@ -255,17 +270,18 @@
     if (defaultField != null) {
       fieldNames.add(defaultField);
     }
-    
 
     Map queries = new HashMap();
-    
-    Set nonWeightedTerms = new HashSet();
+
+    // Set<Term> containing the query terms
+    Set /* <Term> */nonWeightedTerms = new HashSet();
     final boolean mustRewriteQuery = mustRewriteQuery(spanQuery);
     if (mustRewriteQuery) {
       Iterator it = fieldNames.iterator();
-      while(it.hasNext()) {
+      while (it.hasNext()) {
         String field = (String) it.next();
-        final SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(getReaderForField(field));
+        final SpanQuery rewrittenQuery = (SpanQuery) spanQuery
+            .rewrite(getReaderForField(field));
         queries.put(field, rewrittenQuery);
         rewrittenQuery.extractTerms(nonWeightedTerms);
       }
@@ -274,24 +290,31 @@
     }
 
     Iterator it = fieldNames.iterator();
-    List spanPositions = new ArrayList();
-
+    List/* <List<PositionSpan>> */spanPositions = new ArrayList();
+    // MIKE: Initially, iterate over field names; result is populated
+    // spanPositions.
     while (it.hasNext()) {
       String field = (String) it.next();
 
       IndexReader reader = getReaderForField(field);
       final Spans spans;
       if (mustRewriteQuery) {
+        // System.err.println("MIKE: mustRewriteQuery = true");
         spans = ((SpanQuery) queries.get(field)).getSpans(reader);
       } else {
-        spans = spanQuery.getSpans(reader);
+        spans = spanQuery.getSpans(reader); // MIKE: My example hits
+        // this else
       }
-
-      // collect span positions
+      // MIKE: Okay. Now I have the Spans, and I can call getSubSpans() on
+      // that, and then recurse on the result? What's my exit condition to
+      // terminate my recursion?
+      // TODO
       while (spans.next()) {
-        spanPositions.add(new PositionSpan(spans.start(), spans.end() - 1));
+        int depth = 0;
+        // System.err.println("[NEXT] Spans.start() = " + spans.start() +
+        // ", Spans.end() = " + spans.end());
+        handleSubSpans(spans, spanPositions, depth);
       }
-      
     }
 
     if (spanPositions.size() == 0) {
@@ -299,20 +322,38 @@
       return;
     }
 
+    /*
+     * TODO MIKE: Now, iterate over the Query Terms. Shouldn't this consult the
+     * TokenStream to ensure that, within each set of PositionSpan(start, end),
+     * the given Query Term occurs? What about order? Can I assume the correct
+     * order of terms has already been established?
+     */
     for (Iterator iter = nonWeightedTerms.iterator(); iter.hasNext();) {
       Term queryTerm = (Term) iter.next();
-
-      if (fieldNameComparator(queryTerm.field())) {
-        WeightedSpanTerm weightedSpanTerm = (WeightedSpanTerm) terms.get(queryTerm.text());
-
+      
+      if (fieldNameComparator(queryTerm.field())) { // MIKE: Verify field
+        WeightedSpanTerm weightedSpanTerm = (WeightedSpanTerm) terms
+            .get(queryTerm.text());
+        
         if (weightedSpanTerm == null) {
-          weightedSpanTerm = new WeightedSpanTerm(spanQuery.getBoost(), queryTerm.text());
-          weightedSpanTerm.addPositionSpans(spanPositions);
-          weightedSpanTerm.positionSensitive = true;
-          terms.put(queryTerm.text(), weightedSpanTerm);
+          weightedSpanTerm = new WeightedSpanTerm(spanQuery.getBoost(),
+              queryTerm.text());
+          // MIKE: Only add here if valid
+          List/* <PositionSpan> */posSpanList = getPositionSpanListForTerm(
+              spanPositions, queryTerm.text());
+          if (posSpanList.size() > 0) {
+            weightedSpanTerm.addPositionSpans(posSpanList);
+            weightedSpanTerm.positionSensitive = true;
+            terms.put(queryTerm.text(), weightedSpanTerm);
+          }
         } else {
-          if (spanPositions.size() > 0) {
-            weightedSpanTerm.addPositionSpans(spanPositions);
+          // MIKE: Only add here if valid. spanPositions is 2D, but
+          // the resulting List is 1D (flattened)
+          List/* <PositionSpan> */posSpanList = getPositionSpanListForTerm(
+              spanPositions, queryTerm.text());
+          if (posSpanList.size() > 0) {
+            weightedSpanTerm.addPositionSpans(posSpanList);
+            terms.put(queryTerm.text(), weightedSpanTerm);
           }
         }
       }
@@ -320,8 +361,158 @@
   }
 
   /**
-   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms from the supplied <code>Query</code>.
+   * Return a List<PositionSpan> which is valid for the supplied termText
    * 
+   * @param spanPositions
+   * @param termText
+   * @return List (will not return null)
+   */
+  private List getPositionSpanListForTerm(List spanPositions, String termText) {
+    List /* <PositionSpan> */rv = new ArrayList();
+    Iterator/* <List<PositionSpan>> */listOfPosSpanIter = spanPositions
+        .iterator();
+    while (listOfPosSpanIter.hasNext()) {
+      // PositionSpan ps = (PositionSpan) listOfPosSpanIter.next();
+      List listOfPosSpan = (List) listOfPosSpanIter.next();
+      Iterator/* <PositionSpan> */posSpanIter = listOfPosSpan.iterator();
+      while (posSpanIter.hasNext()) {
+        PositionSpan ps = (PositionSpan) posSpanIter.next();
+        if (ps.validTerms.contains(termText)) {
+          rv.add(ps);
+        }
+      }
+    }
+    return rv;
+  }
+
+  /*
+   * List<List<PositionSpan>> spanPositions, where first index is depth and
+   * second is breadth (across a given depth level).
+   * 
+   * List[depth]<List[breadth]<PositionSpan>>
+   */
+  private void handleSubSpans(Spans spans, List spanPositions, int depth)
+      throws IOException {
+    Spans[] subSpans = null;
+    if (spans instanceof NearSpansOrdered) {
+      NearSpansOrdered nso = (NearSpansOrdered) spans;
+      // nso.next();
+      int start = nso.start();
+      int end = nso.end();
+      System.err.println(getIndentForDepth(depth)
+          + "Got NearSpansOrdered (depth = " + depth + ", start = " + start
+          + ", end = " + end + ")");
+      subSpans = nso.getSubSpans();
+      addPositionSpanForDepth(spanPositions, depth, new PositionSpan(start,
+          end - 1));
+    } else if (spans instanceof NearSpansUnordered) {
+      NearSpansUnordered nsu = (NearSpansUnordered) spans;
+      // nsu.next();
+      int start = nsu.start();
+      int end = nsu.end();
+      System.err.println(getIndentForDepth(depth)
+          + "Got NearSpansUnOrdered (depth = " + depth + ", start = " + start
+          + ", end = " + end + ")");
+      subSpans = nsu.getSubSpans();
+      addPositionSpanForDepth(spanPositions, depth, new PositionSpan(start,
+          end - 1));
+    } else if (spans instanceof TermSpans) { // pop
+      TermSpans termSpans = (TermSpans) spans;
+      // TermSpans needs `public Term getTerm()' method ...
+      // termSpans.next(); // Never true, a no-op for TermSpans(?)
+      System.err.println(getIndentForDepth(depth) + "TermSpans(depth = "
+          + depth + ", start = " + termSpans.start() + ", end = "
+          + termSpans.end() + ", term = \"" + termSpans.getTerm().text()
+          + "\")");
+      PositionSpan ps = null;
+      if (0 == depth) {
+        ps = new PositionSpan(termSpans.start(), termSpans.end() - 1);
+        addPositionSpanForDepth(spanPositions, depth, ps);
+      } else {
+        // ps = (PositionSpan) spanPositions.get(depth - 1);
+        List listOfPositionSpan = (List) spanPositions.get(depth - 1);
+        ps = (PositionSpan) listOfPositionSpan
+            .get(listOfPositionSpan.size() - 1);
+      }
+
+      // Add the current Term's text to the Set of valid terms
+      ps.validTerms.add(termSpans.getTerm().text());
+
+      if (0 == depth) {
+        List listOfPositionSpan = (List) spanPositions.get(depth);
+        listOfPositionSpan.add(ps);
+        spanPositions.set(depth, listOfPositionSpan);
+      } else {
+        List listOfPositionSpan = (List) spanPositions.get(depth - 1);
+        listOfPositionSpan.add(ps);
+        spanPositions.set(depth - 1, listOfPositionSpan);
+      }
+      System.err
+          .println(getIndentForDepth(depth) + "Adding span (" + ps.start + ", "
+              + ps.end + ") for term \"" + termSpans.getTerm().text() + "\"");
+      return;
+    } else if (spans instanceof Spans) {
+      /*
+       * System.err.println("MIKE: Spans (start: " + spans.start() + ", end: " +
+       * spans.end() + ")" + spans.toString());
+       */
+      // System.err.println("MIKE: Spans " + spans.toString());
+      subSpans = spans.getSubSpans();
+      if (null != subSpans) {
+        for (int i = 0; i < subSpans.length; i++) {
+          subSpans[i].next();
+          handleSubSpans(subSpans[i], spanPositions, depth);
+        }
+      }
+      return;
+    }
+    if (null != subSpans) {
+      ++depth;
+      for (int i = 0; i < subSpans.length; i++) {
+        subSpans[i].next();
+        handleSubSpans(subSpans[i], spanPositions, depth);
+      }
+    }
+  }
+
+  /**
+   * This takes care of growing the List of List of PositionSpan, where the
+   * index into the first List is depth
+   * 
+   * @param spanPositions
+   * @param depth
+   * @param ps
+   */
+  private void addPositionSpanForDepth(List spanPositions, int depth,
+      PositionSpan ps) {
+    // As necessary, add lists to contain PositionSpans
+    for (int i = spanPositions.size(); i <= depth; i++) {
+      spanPositions.add(new ArrayList());
+    }
+    List listOfPositionSpan = (List) spanPositions.get(depth);
+    listOfPositionSpan.add(ps);
+    spanPositions.set(depth, listOfPositionSpan);
+  }
+
+  // MIKE: Used to grab the text
+  private String getTextFromTokenStream() throws IOException {
+    tokenStream.reset();
+    TermAttribute termAtt = (TermAttribute) tokenStream
+        .addAttribute(TermAttribute.class);
+    StringBuilder sb = new StringBuilder();
+    while (tokenStream.incrementToken()) {
+      if (sb.length() > 0)
+        sb.append(" ");
+      sb.append(termAtt.term());
+    }
+    tokenStream.reset();
+    return sb.toString();
+  }
+
+  /**
+   * Fills a <code>Map</code> with <@link WeightedSpanTerm>s using the terms
+   * from the supplied <code>Query</code>.
+   * 
    * @param terms
    *          Map to place created WeightedSpanTerms in
    * @param query
@@ -336,14 +527,16 @@
       Term queryTerm = (Term) iter.next();
 
       if (fieldNameComparator(queryTerm.field())) {
-        WeightedSpanTerm weightedSpanTerm = new WeightedSpanTerm(query.getBoost(), queryTerm.text());
+        WeightedSpanTerm weightedSpanTerm = new WeightedSpanTerm(query
+            .getBoost(), queryTerm.text());
         terms.put(queryTerm.text(), weightedSpanTerm);
       }
     }
   }
 
   /**
-   * Necessary to implement matches for queries against <code>defaultField</code>
+   * Necessary to implement matches for queries against
+   * <code>defaultField</code>
    */
   private boolean fieldNameComparator(String fieldNameToCheck) {
     boolean rv = fieldName == null || fieldNameToCheck == fieldName
@@ -352,7 +545,8 @@
   }
 
   private IndexReader getReaderForField(String field) throws IOException {
-    if(wrapToCaching && !cachedTokenStream && !(tokenStream instanceof CachingTokenFilter)) {
+    if (wrapToCaching && !cachedTokenStream
+        && !(tokenStream instanceof CachingTokenFilter)) {
       tokenStream = new CachingTokenFilter(tokenStream);
       cachedTokenStream = true;
     }
@@ -370,7 +564,8 @@
   }
 
   /**
-   * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>.
+   * Creates a Map of <code>WeightedSpanTerms</code> from the given
+   * <code>Query</code> and <code>TokenStream</code>.
    * 
    * <p>
    * 
@@ -387,7 +582,8 @@
   }
 
   /**
-   * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>.
+   * Creates a Map of <code>WeightedSpanTerms</code> from the given
+   * <code>Query</code> and <code>TokenStream</code>.
    * 
    * <p>
    * 
@@ -420,8 +616,10 @@
   }
 
   /**
-   * Creates a Map of <code>WeightedSpanTerms</code> from the given <code>Query</code> and <code>TokenStream</code>. Uses a supplied
-   * <code>IndexReader</code> to properly weight terms (for gradient highlighting).
+   * Creates a Map of <code>WeightedSpanTerms</code> from the given
+   * <code>Query</code> and <code>TokenStream</code>. Uses a supplied
+   * <code>IndexReader</code> to properly weight terms (for gradient
+   * highlighting).
    * 
    * <p>
    * 
@@ -436,8 +634,9 @@
    * @return Map of WeightedSpanTerms with quasi tf/idf scores
    * @throws IOException
    */
-  public Map getWeightedSpanTermsWithScores(Query query, TokenStream tokenStream, String fieldName,
-      IndexReader reader) throws IOException {
+  public Map getWeightedSpanTermsWithScores(Query query,
+      TokenStream tokenStream, String fieldName, IndexReader reader)
+      throws IOException {
     if (fieldName != null) {
       this.fieldName = StringHelper.intern(fieldName);
     } else {
@@ -454,14 +653,17 @@
 
     try {
       while (it.hasNext()) {
-        WeightedSpanTerm weightedSpanTerm = (WeightedSpanTerm) terms.get(it.next());
-        int docFreq = reader.docFreq(new Term(fieldName, weightedSpanTerm.term));
+        WeightedSpanTerm weightedSpanTerm = (WeightedSpanTerm) terms.get(it
+            .next());
+        int docFreq = reader
+            .docFreq(new Term(fieldName, weightedSpanTerm.term));
         // docFreq counts deletes
-        if(totalNumDocs < docFreq) {
+        if (totalNumDocs < docFreq) {
           docFreq = totalNumDocs;
         }
         // IDF algorithm taken from DefaultSimilarity class
-        float idf = (float) (Math.log((float) totalNumDocs / (double) (docFreq + 1)) + 1.0);
+        float idf = (float) (Math.log((float) totalNumDocs
+            / (double) (docFreq + 1)) + 1.0);
         weightedSpanTerm.weight *= idf;
       }
     } finally {
@@ -471,64 +673,68 @@
 
     return terms;
   }
-  
 
   private void collectSpanQueryFields(SpanQuery spanQuery, Set fieldNames) {
     if (spanQuery instanceof FieldMaskingSpanQuery) {
-      collectSpanQueryFields(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery(), fieldNames);
+      collectSpanQueryFields(((FieldMaskingSpanQuery) spanQuery)
+          .getMaskedQuery(), fieldNames);
     } else if (spanQuery instanceof SpanFirstQuery) {
-      collectSpanQueryFields(((SpanFirstQuery)spanQuery).getMatch(), fieldNames);
+      collectSpanQueryFields(((SpanFirstQuery) spanQuery).getMatch(),
+          fieldNames);
     } else if (spanQuery instanceof SpanNearQuery) {
-      SpanQuery[] clauses = ((SpanNearQuery)spanQuery).getClauses();
-      for(int i = 0; i < clauses.length; i++) {
+      SpanQuery[] clauses = ((SpanNearQuery) spanQuery).getClauses();
+      for (int i = 0; i < clauses.length; i++) {
         collectSpanQueryFields(clauses[i], fieldNames);
       }
     } else if (spanQuery instanceof SpanNotQuery) {
-      collectSpanQueryFields(((SpanNotQuery)spanQuery).getInclude(), fieldNames);
+      collectSpanQueryFields(((SpanNotQuery) spanQuery).getInclude(),
+          fieldNames);
     } else if (spanQuery instanceof SpanOrQuery) {
-      SpanQuery[] clauses = ((SpanOrQuery)spanQuery).getClauses();
-      for(int i = 0; i < clauses.length; i++) {
+      SpanQuery[] clauses = ((SpanOrQuery) spanQuery).getClauses();
+      for (int i = 0; i < clauses.length; i++) {
         collectSpanQueryFields(clauses[i], fieldNames);
       }
     } else {
       fieldNames.add(spanQuery.getField());
     }
   }
-  
+
   private boolean mustRewriteQuery(SpanQuery spanQuery) {
     if (!expandMultiTermQuery) {
-      return false; // Will throw UnsupportedOperationException in case of a SpanRegexQuery.
+      return false; // Will throw UnsupportedOperationException in case of
+      // a SpanRegexQuery.
     } else if (spanQuery instanceof FieldMaskingSpanQuery) {
-      return mustRewriteQuery(((FieldMaskingSpanQuery)spanQuery).getMaskedQuery());
+      return mustRewriteQuery(((FieldMaskingSpanQuery) spanQuery)
+          .getMaskedQuery());
     } else if (spanQuery instanceof SpanFirstQuery) {
-      return mustRewriteQuery(((SpanFirstQuery)spanQuery).getMatch());
+      return mustRewriteQuery(((SpanFirstQuery) spanQuery).getMatch());
     } else if (spanQuery instanceof SpanNearQuery) {
-      SpanQuery[] clauses = ((SpanNearQuery)spanQuery).getClauses();
-      for(int i = 0; i < clauses.length; i++) {
+      SpanQuery[] clauses = ((SpanNearQuery) spanQuery).getClauses();
+      for (int i = 0; i < clauses.length; i++) {
         if (mustRewriteQuery(clauses[i])) {
           return true;
         }
       }
-      return false; 
+      return false;
     } else if (spanQuery instanceof SpanNotQuery) {
-      SpanNotQuery spanNotQuery = (SpanNotQuery)spanQuery;
-      return mustRewriteQuery(spanNotQuery.getInclude()) || mustRewriteQuery(spanNotQuery.getExclude());
+      SpanNotQuery spanNotQuery = (SpanNotQuery) spanQuery;
+      return mustRewriteQuery(spanNotQuery.getInclude())
+          || mustRewriteQuery(spanNotQuery.getExclude());
     } else if (spanQuery instanceof SpanOrQuery) {
-      SpanQuery[] clauses = ((SpanOrQuery)spanQuery).getClauses();
-      for(int i = 0; i < clauses.length; i++) {  
+      SpanQuery[] clauses = ((SpanOrQuery) spanQuery).getClauses();
+      for (int i = 0; i < clauses.length; i++) {
         if (mustRewriteQuery(clauses[i])) {
           return true;
         }
       }
-      return false; 
+      return false;
     } else if (spanQuery instanceof SpanTermQuery) {
       return false;
     } else {
       return true;
     }
   }
-  
-  
+
   /**
    * This class makes sure that if both position sensitive and insensitive
    * versions of the same term are added, the position insensitive one wins.
@@ -545,40 +751,42 @@
 
     public Object put(Object key, Object value) {
       Object prev = super.put(key, value);
-      if (prev == null) return prev;
-      WeightedSpanTerm prevTerm = (WeightedSpanTerm)prev;
-      WeightedSpanTerm newTerm = (WeightedSpanTerm)value;
+      if (prev == null)
+        return prev;
+      WeightedSpanTerm prevTerm = (WeightedSpanTerm) prev;
+      WeightedSpanTerm newTerm = (WeightedSpanTerm) value;
       if (!prevTerm.positionSensitive) {
         newTerm.positionSensitive = false;
       }
       return prev;
     }
-    
+
   }
-  
+
   private MultiTermQuery copyMultiTermQuery(MultiTermQuery query) {
-    if(query instanceof TermRangeQuery) {
-      TermRangeQuery q = (TermRangeQuery)query;
+    if (query instanceof TermRangeQuery) {
+      TermRangeQuery q = (TermRangeQuery) query;
       q.setBoost(query.getBoost());
-      return new TermRangeQuery(q.getField(), q.getLowerTerm(), q.getUpperTerm(), q.includesLower(), q.includesUpper());
-    } else if(query instanceof WildcardQuery) {
+      return new TermRangeQuery(q.getField(), q.getLowerTerm(), q
+          .getUpperTerm(), q.includesLower(), q.includesUpper());
+    } else if (query instanceof WildcardQuery) {
       MultiTermQuery q = new WildcardQuery(query.getTerm());
       q.setBoost(query.getBoost());
       return q;
-    } else if(query instanceof PrefixQuery) {
+    } else if (query instanceof PrefixQuery) {
       MultiTermQuery q = new PrefixQuery(query.getTerm());
       q.setBoost(q.getBoost());
       return q;
-    } else if(query instanceof FuzzyQuery) {
-      FuzzyQuery q = (FuzzyQuery)query;
+    } else if (query instanceof FuzzyQuery) {
+      FuzzyQuery q = (FuzzyQuery) query;
       q.setBoost(q.getBoost());
-      return new FuzzyQuery(q.getTerm(), q.getMinSimilarity(), q.getPrefixLength());
+      return new FuzzyQuery(q.getTerm(), q.getMinSimilarity(), q
+          .getPrefixLength());
     }
-    
+
     return query;
   }
-  
-  
+
   public boolean getExpandMultiTermQuery() {
     return expandMultiTermQuery;
   }
@@ -586,15 +794,15 @@
   public void setExpandMultiTermQuery(boolean expandMultiTermQuery) {
     this.expandMultiTermQuery = expandMultiTermQuery;
   }
-  
+
   public boolean isCachedTokenStream() {
     return cachedTokenStream;
   }
-  
+
   public TokenStream getTokenStream() {
     return tokenStream;
   }
-  
+
   /**
    * By default, {@link TokenStream}s that are not of the type
    * {@link CachingTokenFilter} are wrapped in a {@link CachingTokenFilter} to
@@ -607,4 +815,14 @@
   public void setWrapIfNotCachingTokenFilter(boolean wrap) {
     this.wrapToCaching = wrap;
   }
+
+  private String getIndentForDepth(int depth) {
+    if (depth == 0)
+      return "";
+    StringBuilder rv = new StringBuilder();
+    for (int i = 0; i < depth; i++) {
+      rv.append("  ");
+    }
+    return rv.toString();
+  }
 }
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java	(revision 908726)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTerm.java	(working copy)
@@ -1,6 +1,5 @@
 package org.apache.lucene.search.highlight;
 
-
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -18,16 +17,18 @@
  * limitations under the License.
  */
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
 
-
 /**
  * Lightweight class to hold term, weight, and positions used for scoring this
  * term.
  */
-public class WeightedSpanTerm extends WeightedTerm{
-  boolean positionSensitive;
+public class WeightedSpanTerm extends WeightedTerm {
+  boolean positionSensitive = false;
   private List positionSpans = new ArrayList();
 
   /**
@@ -51,9 +52,9 @@
 
   /**
    * Checks to see if this term is valid at <code>position</code>.
-   *
+   * 
    * @param position
-   *            to check against valid term postions
+   *          to check against valid term postions
    * @return true iff this term is a hit at this position
    */
   public boolean checkPosition(int position) {
@@ -62,16 +63,23 @@
     // could
     // bail early without checking each PositionSpan.
     Iterator positionSpanIt = positionSpans.iterator();
-
+    boolean rv = false;
     while (positionSpanIt.hasNext()) {
       PositionSpan posSpan = (PositionSpan) positionSpanIt.next();
 
       if (((position >= posSpan.start) && (position <= posSpan.end))) {
-        return true;
+        rv = true;
       }
     }
-
-    return false;
+    final Set termSet = new HashSet();
+    termSet.add("fish");
+    termSet.add("chips");
+    termSet.add("frozen");
+    if (termSet.contains(getTerm())) {
+      System.err.println("checkPosition(" + position + ") is returning " + rv
+          + " for term \"" + getTerm() + "\"");
+    }
+    return rv;
   }
 
   public void addPositionSpans(List positionSpans) {
@@ -91,14 +99,25 @@
   }
 }
 
-
 // Utility class to store a Span
 class PositionSpan {
+
   int start;
   int end;
+  // Set of Term.text which are valid for this span
+  Set /* <String> */validTerms;
 
   public PositionSpan(int start, int end) {
+    this(start, end, new TreeSet());
+  }
+
+  public PositionSpan(int start, int end, Set validTerms) {
     this.start = start;
     this.end = end;
+    this.validTerms = validTerms;
   }
+
+  public boolean contains(String term) {
+    return validTerms.contains(term);
+  }
 }