Index: contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java
===================================================================
--- contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java	(revision 636955)
+++ contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java	(working copy)
@@ -31,12 +31,20 @@
   public static final int DEFAULT_MIN_NGRAM_SIZE = 1;
   public static final int DEFAULT_MAX_NGRAM_SIZE = 2;
 
+  public static final int NO_OPTIMIZE = 0;
+  public static final int QUERY_OPTIMIZE = 1;
+
   private int minGram, maxGram;
-  private int gramSize;
+  
+  private int optimize = 0;
+  
   private int pos = 0;
-  private int inLen;
-  private String inStr;
-  private boolean started = false;
+  private char[] buff;
+  private int buffpos = 0;
+  private int bufflen = 0;
+  private int gramSize = 0;
+  private char[] charRead = new char[1];
+  private char[] queryRead;
 
   /**
    * Creates NGramTokenizer with given min and max n-grams.
@@ -54,6 +62,8 @@
     }
     this.minGram = minGram;
     this.maxGram = maxGram;
+    
+    this.buff = new char[maxGram*2-1];
   }
   /**
    * Creates NGramTokenizer with default min and max n-grams.
@@ -65,26 +75,83 @@
 
   /** Returns the next token in the stream, or null at EOS. */
   public final Token next() throws IOException {
-    if (!started) {
-      started = true;
-      gramSize = minGram;
-      char[] chars = new char[1024];
-      input.read(chars);
-      inStr = new String(chars).trim();  // remove any trailing empty strings 
-      inLen = inStr.length();
+    if(this.optimize==QUERY_OPTIMIZE){
+      if(queryRead==null) queryRead = new char[maxGram];
+      int r=input.read(queryRead);
+      if(r < 0) return null;
+      
+      Token tk=new Token(new String(queryRead,0,r), pos, pos+r);
+      if(pos !=0 ){
+        tk.setPositionIncrement(gramSize);
+      }
+      gramSize = r;
+      pos += r;
+      return tk;
     }
-
-    if (pos+gramSize > inLen) {            // if we hit the end of the string
-      pos = 0;                           // reset to beginning of string
-      gramSize++;                        // increase n-gram size
-      if (gramSize > maxGram)            // we are done
-        return null;
-      if (pos+gramSize > inLen)
-        return null;
+    
+    int increment = 0;
+    if(gramSize < minGram){
+      int insertPos=buffpos;
+      increment = 1;
+      if (pos==0 && gramSize==0) {
+        char[] firstRead = new char[maxGram];
+        bufflen = input.read(firstRead);
+        if(bufflen < 0) return null;
+        
+        for(int i=0; i<bufflen; i++){
+          this.buff[i]=firstRead[i];
+        }
+        firstRead = null;
+      } else {
+        pos++;
+        buffpos++;
+        buffpos %= maxGram;
+        bufflen--;
+        
+        int charReadLen = input.read(charRead);
+        if(charReadLen > 0){
+          bufflen++;
+          for(int i=0; i<2; i++){
+            if((insertPos+maxGram*i) >= buff.length) break;
+            buff[insertPos+maxGram*i]=charRead[0];
+          }
+        }
+      }
+      while(bufflen < minGram){ // catch for charReadLen==0 case
+        int charReadLen = input.read(charRead);
+        if(charReadLen > 0){
+          bufflen++;
+          for(int i=0; i<2; i++){
+            if((insertPos+maxGram*i) >= buff.length) break;
+            buff[insertPos+maxGram*i]=charRead[0];
+          }
+        }
+        if(charReadLen < 0) return null;
+      }
+      gramSize = bufflen;
     }
-    String gram = inStr.substring(pos, pos+gramSize);
-    int oldPos = pos;
-    pos++;
-    return new Token(gram, oldPos, oldPos+gramSize);
+    
+    Token tk = new Token(new String(buff,buffpos,gramSize), pos, pos+gramSize);
+    tk.setPositionIncrement(increment);
+    gramSize--;
+    
+    return tk;
   }
+  
+  public void reset(Reader input) throws IOException {
+    super.reset(input);
+    this.pos=0;
+    this.buffpos = 0;
+    this.gramSize = 0;
+  }
+  
+  /**
+   * Set optimization criteria
+   *
+   * <li><i>NO_OPTIMIZE</i> generates all avaulable tokens
+   * <li><i>QUERY_OPTIMIZE</i> skip some tokens that is nessessary in query phase
+   */
+  public final void setOptimize(int flag){
+    this.optimize=flag;
+  }
 }