Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java	(working copy)
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.core.KeywordTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 
@@ -37,4 +38,9 @@
   public KeywordTokenizer create(Reader input) {
     return new KeywordTokenizer(input);
   }
+  
+  @Override
+  public KeywordTokenizer create(AttributeSource source, Reader input) {
+    return new KeywordTokenizer(source, input, KeywordTokenizer.DEFAULT_BUFFER_SIZE);
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java	(working copy)
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.core.LetterTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -45,4 +46,9 @@
   public LetterTokenizer create(Reader input) {
     return new LetterTokenizer(luceneMatchVersion, input);
   }
+  
+  @Override
+  public LetterTokenizer create(AttributeSource source, Reader input) {
+    return new LetterTokenizer(luceneMatchVersion, source, input);
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java	(working copy)
@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.util.AbstractAnalysisFactory;
 import org.apache.lucene.analysis.util.MultiTermAwareComponent;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -46,6 +47,11 @@
   public LowerCaseTokenizer create(Reader input) {
     return new LowerCaseTokenizer(luceneMatchVersion,input);
   }
+  
+  @Override
+  public LowerCaseTokenizer create(AttributeSource source, Reader input) {
+    return new LowerCaseTokenizer(luceneMatchVersion, source, input);
+  }
 
   @Override
   public AbstractAnalysisFactory getMultiTermComponent() {
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java	(working copy)
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.core.WhitespaceTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -44,4 +45,9 @@
   public WhitespaceTokenizer create(Reader input) {
     return new WhitespaceTokenizer(luceneMatchVersion,input);
   }
+  
+  @Override
+  public WhitespaceTokenizer create(AttributeSource source, Reader input) {
+    return new WhitespaceTokenizer(luceneMatchVersion, source, input);
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilterFactory.java	(working copy)
@@ -39,7 +39,7 @@
 public class PersianCharFilterFactory extends CharFilterFactory implements MultiTermAwareComponent {
 
   @Override
-  public CharFilter create(Reader input) {
+  public PersianCharFilter create(Reader input) {
     return new PersianCharFilter(input);
   }
 
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java	(working copy)
@@ -19,6 +19,7 @@
 
 import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -59,4 +60,9 @@
   public EdgeNGramTokenizer create(Reader input) {
     return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize);
   }
+  
+  @Override
+  public EdgeNGramTokenizer create(AttributeSource source, Reader input) {
+    return new EdgeNGramTokenizer(source, input, side, minGramSize, maxGramSize);
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java	(working copy)
@@ -20,6 +20,7 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.ngram.NGramTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -54,4 +55,11 @@
   public NGramTokenizer create(Reader input) {
     return new NGramTokenizer(input, minGramSize, maxGramSize);
   }
+  
+  /** Creates the {@link TokenStream} of n-grams from the given {@link Reader}
+   * and using the given {@link AttributeSource}. */
+  @Override
+  public NGramTokenizer create(AttributeSource source, Reader input) {
+    return new NGramTokenizer(source, input, minGramSize, maxGramSize);
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilterFactory.java	(working copy)
@@ -55,7 +55,7 @@
   }
 
   @Override
-  public CharFilter create(Reader input) {
+  public PatternReplaceCharFilter create(Reader input) {
     return new PatternReplaceCharFilter( p, replacement, input );
   }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java	(working copy)
@@ -21,6 +21,7 @@
 import org.apache.lucene.analysis.standard.ClassicTokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -50,9 +51,16 @@
   }
 
   @Override
-  public Tokenizer create(Reader input) {
+  public ClassicTokenizer create(Reader input) {
     ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, input); 
     tokenizer.setMaxTokenLength(maxTokenLength);
     return tokenizer;
   }
+  
+  @Override
+  public ClassicTokenizer create(AttributeSource source, Reader input) {
+    ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, source, input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java	(working copy)
@@ -20,6 +20,7 @@
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.StandardTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -54,4 +55,12 @@
     tokenizer.setMaxTokenLength(maxTokenLength);
     return tokenizer;
   }
+  
+  @Override
+  public StandardTokenizer create(AttributeSource source, Reader input) {
+    StandardTokenizer tokenizer
+      = new StandardTokenizer(luceneMatchVersion, source, input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java	(working copy)
@@ -20,6 +20,7 @@
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Map;
@@ -54,4 +55,11 @@
     tokenizer.setMaxTokenLength(maxTokenLength);
     return tokenizer;
   }
+  
+  @Override
+  public UAX29URLEmailTokenizer create(AttributeSource source, Reader input) {
+    UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, source, input); 
+    tokenizer.setMaxTokenLength(maxTokenLength);
+    return tokenizer;
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java	(working copy)
@@ -18,6 +18,7 @@
  */
 
 import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.util.AttributeSource;
 
 import java.io.Reader;
 import java.util.Set;
@@ -63,4 +64,9 @@
 
   /** Creates a TokenStream of the specified input */
   public abstract Tokenizer create(Reader input);
+
+  /** Creates a TokenStream of the specified input using the given AttributeSource */
+  public Tokenizer create(AttributeSource source, Reader input) {
+    throw new UnsupportedOperationException();
+  }
 }
Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java
===================================================================
--- lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java	(working copy)
@@ -18,10 +18,12 @@
  */
 
 import java.io.Reader;
+import java.util.Collections;
 
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
 import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer;
+import org.apache.lucene.util.AttributeSource;
 
 /** 
  * Factory for {@link WikipediaTokenizer}.
@@ -36,7 +38,13 @@
 public class WikipediaTokenizerFactory extends TokenizerFactory {
   // TODO: add support for WikipediaTokenizer's advanced options.
   @Override
-  public Tokenizer create(Reader input) {
+  public WikipediaTokenizer create(Reader input) {
     return new WikipediaTokenizer(input);
   }
+  
+  @Override
+  public WikipediaTokenizer create(AttributeSource source, Reader input) {
+    return new WikipediaTokenizer(source, input, WikipediaTokenizer.TOKENS_ONLY, 
+        Collections.<String>emptySet());
+  }
 }
Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
===================================================================
--- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java	(working copy)
@@ -144,7 +144,7 @@
   }
 
   @Override
-  public Tokenizer create(Reader input) {
+  public ICUTokenizer create(Reader input) {
     assert config != null : "inform must be called first!";
     return new ICUTokenizer(input, config);
   }
Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java	(revision 1436331)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseIterationMarkCharFilterFactory.java	(working copy)
@@ -47,7 +47,7 @@
   private boolean normalizeKana = true;
 
   @Override
-  public CharFilter create(Reader input) {
+  public JapaneseIterationMarkCharFilter create(Reader input) {
     return new JapaneseIterationMarkCharFilter(input, normalizeKanji, normalizeKana);
   }
 
Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java	(working copy)
@@ -89,7 +89,7 @@
   }
   
   @Override
-  public Tokenizer create(Reader input) {
+  public JapaneseTokenizer create(Reader input) {
     return new JapaneseTokenizer(input, userDictionary, discardPunctuation, mode);
   }
   
Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java
===================================================================
--- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java	(working copy)
@@ -22,6 +22,7 @@
 import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.cn.smart.SentenceTokenizer;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
 
 /**
  * Factory for the SmartChineseAnalyzer {@link SentenceTokenizer}
@@ -29,7 +30,12 @@
  */
 public class SmartChineseSentenceTokenizerFactory extends TokenizerFactory {
   @Override
-  public Tokenizer create(Reader input) {
+  public SentenceTokenizer create(Reader input) {
     return new SentenceTokenizer(input);
   }
+  
+  @Override
+  public SentenceTokenizer create(AttributeSource source, Reader input) {
+    return new SentenceTokenizer(source, input);
+  }
 }
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java	(working copy)
@@ -43,7 +43,7 @@
   }
 
   @Override
-  public Tokenizer create(Reader input) {
+  public UIMAAnnotationsTokenizer create(Reader input) {
     return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, input);
   }
 }
Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java
===================================================================
--- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java	(revision 1436331)
+++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java	(working copy)
@@ -45,7 +45,7 @@
   }
 
   @Override
-  public Tokenizer create(Reader input) {
+  public UIMATypeAwareAnnotationsTokenizer create(Reader input) {
     return new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, input);
   }
 }
Index: solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java
===================================================================
--- solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java	(revision 1436331)
+++ solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java	(working copy)
@@ -41,7 +41,7 @@
   }
 
   @Override
-  public CharFilter create(Reader input) {
+  public MockCharFilter create(Reader input) {
     return new MockCharFilter(input, remainder);
   }
 }
Index: solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java
===================================================================
--- solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java	(revision 1436331)
+++ solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java	(working copy)
@@ -55,7 +55,7 @@
 
 
   @Override
-  public Tokenizer create(Reader input) {
+  public MockTokenizer create(Reader input) {
     MockTokenizer t = new MockTokenizer(input, pattern, false);
     t.setEnableChecks(enableChecks);
     return t;