Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java	(revision 820786)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java	(working copy)
@@ -254,39 +254,54 @@
 					throw new InvalidTokenOffsetsException("Token "+ termAtt.term()
 							+" exceeds length of provided text sized "+text.length());
 				}
-				if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct()))
+				if (tokenGroup.numTokens > 0)
 				{
-					//the current token is distinct from previous tokens -
-					// markup the cached token group info
-					startOffset = tokenGroup.matchStartOffset;
-					endOffset = tokenGroup.matchEndOffset;
-					tokenText = text.substring(startOffset, endOffset);
-					String markedUpText=formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);
-					//store any whitespace etc from between this and last group
-					if (startOffset > lastEndOffset)
-						newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));
-					newText.append(markedUpText);
-					lastEndOffset=Math.max(endOffset, lastEndOffset);
-					tokenGroup.clear();
+					if (tokenGroup.isDistinct())
+					{
+						//the current token is distinct from previous tokens -
+						// markup the cached token group info
+						startOffset = tokenGroup.matchStartOffset;
+						endOffset = tokenGroup.matchEndOffset;
+						tokenText = text.substring(startOffset, endOffset);
+						String markedUpText = formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);
+						//store any whitespace etc from between this and last group
+						if (startOffset > lastEndOffset)
+						{
+							newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));
+						}
+						newText.append(markedUpText);
+						lastEndOffset = Math.max(endOffset, lastEndOffset);
+						tokenGroup.clear();
 
-					//check if current token marks the start of a new fragment
-					if(textFragmenter.isNewFragment())
+						//check if current token marks the start of a new fragment
+						if (textFragmenter.isNewFragment())
+						{
+							currentFrag.setScore(fragmentScorer.getFragmentScore());
+							//record stats for a new fragment
+							currentFrag.textEndPos = newText.length();
+							currentFrag = new TextFragment(newText, newText.length(), docFrags.size());
+							fragmentScorer.startFragment(currentFrag);
+							docFrags.add(currentFrag);
+						}
+					} 
+					else
 					{
-						currentFrag.setScore(fragmentScorer.getFragmentScore());
-						//record stats for a new fragment
-						currentFrag.textEndPos = newText.length();
-						currentFrag =new TextFragment(newText, newText.length(), docFrags.size());
-						fragmentScorer.startFragment(currentFrag);
-						docFrags.add(currentFrag);
+						// next token overlaps tokenGroup
+						float score = fragmentScorer.getTokenScore();
+						if (score != 0)
+						{
+							// token has non-zero score, so add it to the group
+							tokenGroup.addToken(score);
+						} 
+						else
+						{
+							// token has zero score: just skip it
+						}
+						continue;
 					}
 				}
 
 				tokenGroup.addToken(fragmentScorer.getTokenScore());
-
-//				if(lastEndOffset>maxDocBytesToAnalyze)
-//				{
-//					break;
-//				}
 			}
 			currentFrag.setScore(fragmentScorer.getFragmentScore());