Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java	(revision 813527)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java	(working copy)
@@ -285,7 +285,7 @@
       
         if (length > 0) {
           termAtt.setTermBuffer(buffer, 0, length);
-          offsetAtt.setOffset(input.correctOffset(start), input.correctOffset(start+length));
+          offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
           typeAtt.setType(TOKEN_TYPE_NAMES[tokenType]);
           return true;
         } else if (dataLen == -1) {
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java	(revision 813527)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java	(working copy)
@@ -104,7 +104,7 @@
             //System.out.println(new String(buffer, 0,
             //length));
           termAtt.setTermBuffer(buffer, 0, length);
-          offsetAtt.setOffset(input.correctOffset(start), input.correctOffset(start+length));
+          offsetAtt.setOffset(correctOffset(start), correctOffset(start+length));
           return true;
         }
         else
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java	(revision 813527)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java	(working copy)
@@ -207,7 +207,7 @@
     int start = side == Side.FRONT ? 0 : inLen - gramSize;
     int end = start + gramSize;
     termAtt.setTermBuffer(inStr, start, gramSize);
-    offsetAtt.setOffset(input.correctOffset(start), input.correctOffset(end));
+    offsetAtt.setOffset(correctOffset(start), correctOffset(end));
     gramSize++;
     return true;
   }
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java	(revision 813527)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java	(working copy)
@@ -124,7 +124,7 @@
     int oldPos = pos;
     pos++;
     termAtt.setTermBuffer(inStr, oldPos, gramSize);
-    offsetAtt.setOffset(input.correctOffset(oldPos), input.correctOffset(oldPos+gramSize));
+    offsetAtt.setOffset(correctOffset(oldPos), correctOffset(oldPos+gramSize));
     return true;
   }
   
Index: contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java
===================================================================
--- contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java	(revision 813527)
+++ contrib/analyzers/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java	(working copy)
@@ -116,7 +116,7 @@
       return false;
     else {
       termAtt.setTermBuffer(buffer.toString());
-      offsetAtt.setOffset(input.correctOffset(tokenStart), input.correctOffset(tokenEnd));
+      offsetAtt.setOffset(correctOffset(tokenStart), correctOffset(tokenEnd));
       typeAtt.setType("sentence");
       return true;
     }
Index: contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
===================================================================
--- contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java	(revision 813527)
+++ contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java	(working copy)
@@ -202,7 +202,7 @@
         return false;
       
       termAtt.setTermBuffer(snippet, startTerm, lenTerm);
-      offsetAtt.setOffset(startOffset, startOffset + lenTerm);
+      offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
       return true;
     }
 
Index: contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java
===================================================================
--- contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java	(revision 813527)
+++ contrib/wikipedia/src/java/org/apache/lucene/wikipedia/analysis/WikipediaTokenizer.java	(working copy)
@@ -267,7 +267,7 @@
     //trim the buffer
     String s = buffer.toString().trim();
     termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
-    offsetAtt.setOffset(input.correctOffset(theStart), input.correctOffset(theStart + s.length()));
+    offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
     flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
     //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
     if (tmpTokType != WikipediaTokenizerImpl.YYEOF){
@@ -305,7 +305,7 @@
     //trim the buffer
     String s = buffer.toString().trim();
     termAtt.setTermBuffer(s.toCharArray(), 0, s.length());
-    offsetAtt.setOffset(input.correctOffset(theStart), input.correctOffset(theStart + s.length()));
+    offsetAtt.setOffset(correctOffset(theStart), correctOffset(theStart + s.length()));
     flagsAtt.setFlags(UNTOKENIZED_TOKEN_FLAG);
     //The way the loop is written, we will have proceeded to the next token.  We need to pushback the scanner to lastPos
     if (tmpTokType != WikipediaTokenizerImpl.YYEOF){
@@ -318,7 +318,7 @@
   private void setupToken() {
     scanner.getText(termAtt);
     final int start = scanner.yychar();
-    offsetAtt.setOffset(input.correctOffset(start), input.correctOffset(start + termAtt.termLength()));
+    offsetAtt.setOffset(correctOffset(start), correctOffset(start + termAtt.termLength()));
   }
 
   /*