Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java
===================================================================
--- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java	(revision 935242)
+++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java	(working copy)
@@ -34,7 +34,7 @@
     protected int index = 0;
     protected Token[] testToken;
     
-    private TermAttribute termAtt;
+    private CharTermAttribute termAtt;
     private OffsetAttribute offsetAtt;
     private PositionIncrementAttribute posIncrAtt;
     private TypeAttribute typeAtt;
@@ -42,7 +42,7 @@
     public TestTokenStream(Token[] testToken) {
       super();
       this.testToken = testToken;
-      this.termAtt = addAttribute(TermAttribute.class);
+      this.termAtt = addAttribute(CharTermAttribute.class);
       this.offsetAtt = addAttribute(OffsetAttribute.class);
       this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
       this.typeAtt = addAttribute(TypeAttribute.class);
@@ -53,7 +53,7 @@
       clearAttributes();
       if (index < testToken.length) {
         Token t = testToken[index++];
-        termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength());
+        termAtt.copyBuffer(t.buffer(), 0, t.length());
         offsetAtt.setOffset(t.startOffset(), t.endOffset());
         posIncrAtt.setPositionIncrement(t.getPositionIncrement());
         typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE);
@@ -103,19 +103,22 @@
     createToken("please divide", 0, 13),
     createToken("divide", 7, 13),
     createToken("divide _", 7, 19),
-    createToken("_", 19, 19),
     createToken("_ sentence", 19, 27),
     createToken("sentence", 19, 27),
     createToken("sentence _", 19, 33),
-    createToken("_", 33, 33),
     createToken("_ shingles", 33, 39),
     createToken("shingles", 33, 39),
   };
 
   public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES = new int[] {
-    1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
+    1, 0, 1, 0, 1, 1, 0, 1, 1
   };
 
+  private static final String[] BI_GRAM_TYPES_WITH_HOLES = {
+    "word", "shingle", 
+    "word", "shingle", "shingle", "word", "shingle", "shingle", "word"
+  };
+
   public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {
     createToken("please divide", 0, 13),
     createToken("divide this", 7, 18),
@@ -642,18 +645,157 @@
     "word"
   };
   
+  public static final Token[] TEST_TOKEN_POS_INCR_EQUAL_TO_N = new Token[] {
+    createToken("please", 0, 6),
+    createToken("divide", 7, 13),
+    createToken("this", 14, 18),
+    createToken("sentence", 29, 37, 3),
+    createToken("into", 38, 42),
+    createToken("shingles", 43, 49),
+  };
+
+  public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N = new Token[] {
+    createToken("please", 0, 6),
+    createToken("please divide", 0, 13),
+    createToken("please divide this", 0, 18),
+    createToken("divide", 7, 13),
+    createToken("divide this", 7, 18),
+    createToken("divide this _", 7, 29),
+    createToken("this", 14, 18),
+    createToken("this _", 14, 29),
+    createToken("this _ _", 14, 29),
+    createToken("_ _ sentence", 29, 37),
+    createToken("_ sentence", 29, 37),
+    createToken("_ sentence into", 29, 42),
+    createToken("sentence", 29, 37),
+    createToken("sentence into", 29, 42),
+    createToken("sentence into shingles", 29, 49),
+    createToken("into", 38, 42),
+    createToken("into shingles", 38, 49),
+    createToken("shingles", 43, 49)
+  };
+  
+  public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N = new int[] {
+    1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1
+  };
+  
+  public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N = new String[] {
+    "word", "shingle", "shingle",
+    "word", "shingle", "shingle",
+    "word", "shingle", "shingle",
+    "shingle", "shingle", "shingle", "word", "shingle", "shingle",
+    "word", "shingle",
+    "word"
+  };
+  
+  public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new Token[] {
+    createToken("please divide", 0, 13),
+    createToken("please divide this", 0, 18),
+    createToken("divide this", 7, 18),
+    createToken("divide this _", 7, 29),
+    createToken("this _", 14, 29),
+    createToken("this _ _", 14, 29),
+    createToken("_ _ sentence", 29, 37),
+    createToken("_ sentence", 29, 37),
+    createToken("_ sentence into", 29, 42),
+    createToken("sentence into", 29, 42),
+    createToken("sentence into shingles", 29, 49),
+    createToken("into shingles", 38, 49),
+  };
+
+  public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new int[] {
+    1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1
+  };
+
+  public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new String[] {
+    "shingle", "shingle",
+    "shingle", "shingle",
+    "shingle", "shingle",
+    "shingle", "shingle", "shingle",
+    "shingle", "shingle",
+    "shingle",
+  };
+
+  public static final Token[] TEST_TOKEN_POS_INCR_GREATER_THAN_N = new Token[] {
+    createToken("please", 0, 6),
+    createToken("divide", 57, 63, 8),
+    createToken("this", 64, 68),
+    createToken("sentence", 69, 77),
+    createToken("into", 78, 82),
+    createToken("shingles", 83, 89),
+  };
+  
+  public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N = new Token[] {
+    createToken("please", 0, 6),
+    createToken("please _", 0, 57),
+    createToken("please _ _", 0, 57),
+    createToken("_ _ divide", 57, 63),
+    createToken("_ divide", 57, 63),
+    createToken("_ divide this", 57, 68),
+    createToken("divide", 57, 63),
+    createToken("divide this", 57, 68),
+    createToken("divide this sentence", 57, 77),
+    createToken("this", 64, 68),
+    createToken("this sentence", 64, 77),
+    createToken("this sentence into", 64, 82),
+    createToken("sentence", 69, 77),
+    createToken("sentence into", 69, 82),
+    createToken("sentence into shingles", 69, 89),
+    createToken("into", 78, 82),
+    createToken("into shingles", 78, 89),
+    createToken("shingles", 83, 89)
+  };
+  
+  public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N = new int[] {
+    1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
+  };
+  public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N = new String[] {
+    "word", "shingle", "shingle",
+    "shingle",
+    "shingle", "shingle", 
+    "word", "shingle", "shingle",
+    "word", "shingle", "shingle",
+    "word", "shingle", "shingle",
+    "word", "shingle",
+    "word"
+  };
+  
+  public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new Token[] {
+    createToken("please _", 0, 57),
+    createToken("please _ _", 0, 57),
+    createToken("_ _ divide", 57, 63),
+    createToken("_ divide", 57, 63),
+    createToken("_ divide this", 57, 68),
+    createToken("divide this", 57, 68),
+    createToken("divide this sentence", 57, 77),
+    createToken("this sentence", 64, 77),
+    createToken("this sentence into", 64, 82),
+    createToken("sentence into", 69, 82),
+    createToken("sentence into shingles", 69, 89),
+    createToken("into shingles", 78, 89),
+  };
+
+  public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new int[] {
+    1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1
+  };
+
+  public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new String[] {
+    "shingle", "shingle",
+    "shingle", "shingle",
+    "shingle", "shingle",
+    "shingle", "shingle", "shingle", "shingle", "shingle",
+    "shingle",
+  };
+
   @Override
   protected void setUp() throws Exception {
     super.setUp();
     testTokenWithHoles = new Token[] {
       createToken("please", 0, 6),
       createToken("divide", 7, 13),
-      createToken("sentence", 19, 27),
-      createToken("shingles", 33, 39),
+      createToken("sentence", 19, 27, 2),
+      createToken("shingles", 33, 39, 2),
     };
-
-    testTokenWithHoles[2].setPositionIncrement(2);
-    testTokenWithHoles[3].setPositionIncrement(2);
   }
 
   /*
@@ -667,7 +809,8 @@
 
   public void testBiGramFilterWithHoles() throws IOException {
     this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES,
-                           BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES,
+                           BI_GRAM_POSITION_INCREMENTS_WITH_HOLES, 
+                           BI_GRAM_TYPES_WITH_HOLES, 
                            true);
   }
 
@@ -832,8 +975,32 @@
                            TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR, 
                            TRI_GRAM_TYPES_NULL_SEPARATOR, true);
   }
+
+  public void testPositionIncrementEqualToN() throws IOException {
+    this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N,
+                           TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N, 
+                           TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N, true);
+  }
   
+  public void testPositionIncrementEqualToNWithoutUnigrams() throws IOException {
+    this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS,
+                           TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, 
+                           TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, false);
+  }
   
+  
+  public void testPositionIncrementGreaterThanN() throws IOException {
+    this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N,
+                           TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N, 
+                           TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N, true);
+  }
+  
+  public void testPositionIncrementGreaterThanNWithoutUnigrams() throws IOException {
+    this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS,
+                           TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, 
+                           TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, false);
+  }
+  
   public void testReset() throws Exception {
     Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence"));
     TokenStream filter = new ShingleFilter(wsTokenizer, 2);
@@ -896,18 +1063,24 @@
     int endOffsets[] = new int[tokensToCompare.length];
     
     for (int i = 0; i < tokensToCompare.length; i++) {
-      text[i] = tokensToCompare[i].term();
+      text[i] = new String(tokensToCompare[i].buffer(),0, tokensToCompare[i].length());
       startOffsets[i] = tokensToCompare[i].startOffset();
       endOffsets[i] = tokensToCompare[i].endOffset();
     }
     
     assertTokenStreamContents(filter, text, startOffsets, endOffsets, types, positionIncrements);
   }
+  
+  private static Token createToken(String term, int start, int offset) {
+    return createToken(term, start, offset, 1);
+  }
 
-  private static Token createToken(String term, int start, int offset)
+  private static Token createToken
+    (String term, int start, int offset, int positionIncrement)
   {
     Token token = new Token(start, offset);
-    token.setTermBuffer(term);
+    token.copyBuffer(term.toCharArray(), 0, term.length());
+    token.setPositionIncrement(positionIncrement);
     return token;
   }
 }
Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java
===================================================================
--- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java	(revision 935242)
+++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java	(working copy)
@@ -24,8 +24,10 @@
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
 
 
 /**
@@ -66,7 +68,6 @@
    */
   public static final String TOKEN_SEPARATOR = " ";
 
-
   /**
    * The sequence of input stream tokens (or filler tokens, if necessary)
    * that will be composed to form output shingles.
@@ -80,9 +81,9 @@
   private CircularSequence gramSize;
 
   /**
-   * Shingle text is composed here.
+   * Shingle and unigram text is composed here.
    */
-  private StringBuilder shingleBuilder = new StringBuilder();
+  private StringBuilder gramBuilder = new StringBuilder();
 
   /**
    * The token type attribute value to use - default is "shingle"
@@ -111,21 +112,35 @@
   private int minShingleSize;
 
   /**
-   * The remaining number of filler tokens inserted into the input stream
+   * The remaining number of filler tokens to be inserted into the input stream
    * from which shingles are composed, to handle position increments greater
    * than one.
    */
   private int numFillerTokensToInsert;
 
   /**
-   * The next input stream token.
+   * When the next input stream token has a position increment greater than
+   * one, it is stored in this field until sufficient filler tokens have been
+   * inserted to account for the position increment. 
    */
   private State nextInputStreamToken;
+
+  /**
+   * True unless a non-filler token has been appended to {@link #gramBuilder}.
+   */
+  private boolean isFillerOnly = true;
+
+  /**
+   * Whether at least one unigram or shingle has been output at the current 
+   * position.
+   */
+  private boolean isOutputHere = false;
   
-  private final TermAttribute termAtt;
+  private final CharTermAttribute charTermAtt;
   private final OffsetAttribute offsetAtt;
   private final PositionIncrementAttribute posIncrAtt;
   private final TypeAttribute typeAtt;
+  private final FillerAttribute fillerAtt;
 
 
   /**
@@ -140,10 +155,11 @@
     super(input);
     setMaxShingleSize(maxShingleSize);
     setMinShingleSize(minShingleSize);
-    this.termAtt = addAttribute(TermAttribute.class);
+    this.charTermAtt = addAttribute(CharTermAttribute.class);
     this.offsetAtt = addAttribute(OffsetAttribute.class);
     this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
     this.typeAtt = addAttribute(TypeAttribute.class);
+    this.fillerAtt = addAttribute(FillerAttribute.class);
   }
 
   /**
@@ -241,26 +257,33 @@
     this.tokenSeparator = null == tokenSeparator ? "" : tokenSeparator;
   }
 
-  /* (non-Javadoc)
-   * @see org.apache.lucene.analysis.TokenStream#next()
-   */
   @Override
   public final boolean incrementToken() throws IOException {
-    boolean tokenAvailable = false; 
-    if (gramSize.atMinValue() || inputWindow.size() < gramSize.getValue()) {
-      shiftInputWindow();
-    }
-    if ( ! inputWindow.isEmpty()) {
-      restoreState(inputWindow.getFirst());
-      if (1 == gramSize.getValue()) {
-        posIncrAtt.setPositionIncrement(1);
+    boolean tokenAvailable;
+    do {
+      tokenAvailable = false;
+      if (gramSize.atMinValue() || inputWindow.size() < gramSize.getValue()) {
+        shiftInputWindow();
+      }
+      if (inputWindow.size() >= gramSize.getValue()) {
+        restoreState(inputWindow.getFirst());
+        if (1 == gramSize.getValue()) {
+          if (fillerAtt.isFiller()) { // don't output filler unigrams
+            gramSize.advance();
+            getNextShingle();
+          } else {
+            isFillerOnly = false;
+            posIncrAtt.setPositionIncrement(1);
+          }
+        } else {
+          getNextShingle();
+        }
         gramSize.advance();
         tokenAvailable = true;
-      } else if (inputWindow.size() >= gramSize.getValue()) {
-        getNextShingle();
-        gramSize.advance();
-        tokenAvailable = true;
       }
+    } while (isFillerOnly && tokenAvailable); // No filler-only grams
+    if (tokenAvailable) {
+      isOutputHere = true;
     }
     return tokenAvailable;
   }
@@ -278,24 +301,28 @@
     if (gramSize.getValue() == minShingleSize) {
       // Clear the shingle text buffer if this is the first shingle
       // at the current position in the input stream.
-      shingleBuilder.setLength(0);
+      gramBuilder.setLength(0);
+      isFillerOnly = true;
       minTokNum = 0;
     }
     for (int tokNum = minTokNum ; tokNum < gramSize.getValue() ; ++tokNum) {
       if (tokNum > 0) {
-        shingleBuilder.append(tokenSeparator);
+        gramBuilder.append(tokenSeparator);
       }
       restoreState(inputWindow.get(tokNum));
-      shingleBuilder.append(termAtt.termBuffer(), 0, termAtt.termLength());
+      gramBuilder.append(charTermAtt.buffer(), 0, charTermAtt.length());
+      if (isFillerOnly && ! fillerAtt.isFiller()) {
+        isFillerOnly = false;
+      }
     }
-    char[] termBuffer = termAtt.termBuffer();
-    int termLength = shingleBuilder.length();
+    char[] termBuffer = charTermAtt.buffer();
+    int termLength = gramBuilder.length();
     if (termBuffer.length < termLength) {
-      termBuffer = termAtt.resizeTermBuffer(termLength);
+      termBuffer = charTermAtt.resizeBuffer(termLength);
     }
-    shingleBuilder.getChars(0, termLength, termBuffer, 0);
-    termAtt.setTermLength(termLength);
-    posIncrAtt.setPositionIncrement(gramSize.atMinValue() ? 1 : 0);
+    gramBuilder.getChars(0, termLength, termBuffer, 0);
+    charTermAtt.setLength(termLength);
+    posIncrAtt.setPositionIncrement(isOutputHere ? 0 : 1);
     typeAtt.setType(tokenType);
     offsetAtt.setOffset(startOffset, offsetAtt.endOffset());
   }
@@ -319,8 +346,13 @@
       success = true;
     } else if (input.incrementToken()) {
       if (posIncrAtt.getPositionIncrement() > 1) {
-        numFillerTokensToInsert = posIncrAtt.getPositionIncrement() - 1;
+        // Each output shingle must contain at least one input token, 
+        // so no more than (maxShingleSize - 1) filler tokens will be inserted.
+        numFillerTokensToInsert 
+          = Math.min(posIncrAtt.getPositionIncrement() - 1, maxShingleSize - 1);
         insertFillerToken();
+      } else {
+        fillerAtt.setFiller(false);
       }
       success = true;
     }
@@ -340,7 +372,8 @@
     --numFillerTokensToInsert;
     // A filler token occupies no space
     offsetAtt.setOffset(offsetAtt.startOffset(), offsetAtt.startOffset());
-    termAtt.setTermBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.length);
+    charTermAtt.copyBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.length);
+    fillerAtt.setFiller(true);
   }
 
   /**
@@ -354,13 +387,11 @@
     if (inputWindow.size() > 0) {
       inputWindow.removeFirst();
     }
-    while (getNextToken()) {
+    while (inputWindow.size() < maxShingleSize && getNextToken()) {
       inputWindow.add(captureState());
-      if (inputWindow.size() == maxShingleSize) {
-        break;
-      }
     }
     gramSize.reset();
+    isOutputHere = false;
   }
 
   @Override
@@ -369,6 +400,8 @@
     gramSize.reset();
     inputWindow.clear();
     numFillerTokensToInsert = 0;
+    isOutputHere = false;
+    isFillerOnly = true;
   }
 
 
@@ -405,10 +438,8 @@
      * <b>{ [ 1, ] {@link #minShingleSize} [ , ... , {@link #maxShingleSize} ] }</b>.
      * <p>1 is included in the circular sequence only if 
      * {@link #outputUnigrams} = true.
-     * 
-     * @return the next member in the circular sequence
      */
-    public int advance() {
+    public void advance() {
       if (value == 1) {
         value = minShingleSize;
       } else if (value == maxShingleSize) {
@@ -416,7 +447,6 @@
       } else {
         ++value;
       }
-      return value;
     }
 
     /**
@@ -444,4 +474,49 @@
       return value == minValue;
     }
   }
+
+  /**
+   * This attribute is used to indicate whether an input token is a "filler"
+   * token generated by ShingleFilter for position increments greater than one.
+   */
+  public interface FillerAttribute extends Attribute {
+    public void setFiller(boolean filler);
+    public boolean isFiller();
+  }
+  
+  public static final class FillerAttributeImpl extends AttributeImpl
+    implements FillerAttribute {
+
+    private boolean filler = false;
+
+    public void setFiller(boolean filler) {
+      this.filler = filler;
+    }
+
+    public boolean isFiller() {
+      return filler;
+    }
+
+    @Override
+    public void clear() {
+      filler = false;
+    }
+
+    @Override
+    public void copyTo(AttributeImpl target) {
+      FillerAttribute attr = (FillerAttribute)target;
+      attr.setFiller(filler);
+    }
+
+    @Override
+    public boolean equals(Object other) {
+      return other instanceof FillerAttributeImpl
+             && ((FillerAttributeImpl)other).filler == this.filler;
+    }
+
+    @Override
+    public int hashCode() {
+      return this.filler ? -1 : Integer.MAX_VALUE;
+    }
+  }
 }