Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java (revision 1471344) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramFilterFactory.java (working copy) @@ -47,6 +47,6 @@ @Override public NGramTokenFilter create(TokenStream input) { - return new NGramTokenFilter(input, minGramSize, maxGramSize); + return new NGramTokenFilter(luceneMatchVersion, input, minGramSize, maxGramSize); } } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (revision 1471344) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (working copy) @@ -23,15 +23,25 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.Version; /** * Tokenizes the input into n-grams of the given size(s). + * + *

You must specify the required {@link Version} + * compatibility when creating NGramTokenFilter: + *

*/ public final class NGramTokenFilter extends TokenFilter { public static final int DEFAULT_MIN_NGRAM_SIZE = 1; public static final int DEFAULT_MAX_NGRAM_SIZE = 2; private int minGram, maxGram; + private int posInc; private char[] curTermBuffer; private int curTermLength; @@ -43,14 +53,17 @@ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute posAtt; /** * Creates NGramTokenFilter with given min and max n-grams. + * @param matchVersion Lucene version to enable correct position increments. + * See above for details. * @param input {@link TokenStream} holding the input to be tokenized * @param minGram the smallest n-gram to generate * @param maxGram the largest n-gram to generate */ - public NGramTokenFilter(TokenStream input, int minGram, int maxGram) { + public NGramTokenFilter(Version matchVersion, TokenStream input, int minGram, int maxGram) { super(input); if (minGram < 1) { throw new IllegalArgumentException("minGram must be greater than zero"); @@ -60,16 +73,48 @@ } this.minGram = minGram; this.maxGram = maxGram; + posAtt = matchVersion.onOrAfter(Version.LUCENE_44) ? addAttribute(PositionIncrementAttribute.class) : new PositionIncrementAttribute() { + @Override + public void setPositionIncrement(int positionIncrement) {} + @Override + public int getPositionIncrement() { + return 0; + } + }; } - + /** * Creates NGramTokenFilter with default min and max n-grams. + * @param matchVersion Lucene version to enable correct position increments. + * See above for details. * @param input {@link TokenStream} holding the input to be tokenized */ - public NGramTokenFilter(TokenStream input) { - this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE); + public NGramTokenFilter(Version matchVersion, TokenStream input) { + this(matchVersion, input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE); } +// /** +// * Creates NGramTokenFilter with given min and max n-grams. +// * @param input {@link TokenStream} holding the input to be tokenized +// * @param minGram the smallest n-gram to generate +// * @param maxGram the largest n-gram to generate +// * @deprecated use {@link #NGramTokenFilter(Version, TokenStream, int, int)} instead +// */ +// @Deprecated +// public NGramTokenFilter(TokenStream input, int minGram, int maxGram) { +// this(Version.LUCENE_43, input, minGram, maxGram); +// } +// +// /** +// * Creates NGramTokenFilter with default min and max n-grams. +// * @param input {@link TokenStream} holding the input to be tokenized +// * @deprecated use {@link #NGramTokenFilter(Version, TokenStream)} instead +// */ +// @Deprecated +// public NGramTokenFilter(TokenStream input) { +// this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE); +// } + /** Returns the next token in the stream, or null at EOS. */ @Override public final boolean incrementToken() throws IOException { @@ -78,6 +123,7 @@ if (!input.incrementToken()) { return false; } else { + posInc += posAtt.getPositionIncrement(); curTermBuffer = termAtt.buffer().clone(); curTermLength = termAtt.length(); curGramSize = minGram; @@ -92,6 +138,8 @@ while (curGramSize <= maxGram) { while (curPos+curGramSize <= curTermLength) { // while there is input clearAttributes(); + posAtt.setPositionIncrement(posInc); + posInc = 0; termAtt.copyBuffer(curTermBuffer, curPos, curGramSize); if (hasIllegalOffsets) { offsetAtt.setOffset(tokStart, tokEnd); @@ -111,6 +159,7 @@ @Override public void reset() throws IOException { super.reset(); + posInc = 0; curTermBuffer = null; } } Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (revision 1471344) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenFilterTest.java (working copy) @@ -46,7 +46,7 @@ public void testInvalidInput() throws Exception { boolean gotException = false; try { - new NGramTokenFilter(input, 2, 1); + new NGramTokenFilter(TEST_VERSION_CURRENT, input, 2, 1); } catch (IllegalArgumentException e) { gotException = true; } @@ -56,7 +56,7 @@ public void testInvalidInput2() throws Exception { boolean gotException = false; try { - new NGramTokenFilter(input, 0, 1); + new NGramTokenFilter(TEST_VERSION_CURRENT, input, 0, 1); } catch (IllegalArgumentException e) { gotException = true; } @@ -64,42 +64,44 @@ } public void testUnigrams() throws Exception { - NGramTokenFilter filter = new NGramTokenFilter(input, 1, 1); - assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 1, 1); + assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, new int[]{1,0,0,0,0}); } public void testBigrams() throws Exception { - NGramTokenFilter filter = new NGramTokenFilter(input, 2, 2); - assertTokenStreamContents(filter, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 2, 2); + assertTokenStreamContents(filter, new String[]{"ab","bc","cd","de"}, new int[]{0,1,2,3}, new int[]{2,3,4,5}, new int[]{1,0,0,0}); } public void testNgrams() throws Exception { - NGramTokenFilter filter = new NGramTokenFilter(input, 1, 3); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 1, 3); assertTokenStreamContents(filter, new String[]{"a","b","c","d","e", "ab","bc","cd","de", "abc","bcd","cde"}, new int[]{0,1,2,3,4, 0,1,2,3, 0,1,2}, new int[]{1,2,3,4,5, 2,3,4,5, 3,4,5}, - null, null, null, null, false + null, + new int[]{1,0,0,0,0, 0,0,0,0, 0,0,0}, + null, null, false ); } public void testOversizedNgrams() throws Exception { - NGramTokenFilter filter = new NGramTokenFilter(input, 6, 7); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 6, 7); assertTokenStreamContents(filter, new String[0], new int[0], new int[0]); } public void testSmallTokenInStream() throws Exception { input = new MockTokenizer(new StringReader("abc de fgh"), MockTokenizer.WHITESPACE, false); - NGramTokenFilter filter = new NGramTokenFilter(input, 3, 3); - assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10}); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, input, 3, 3); + assertTokenStreamContents(filter, new String[]{"abc","fgh"}, new int[]{0,7}, new int[]{3,10}, new int[] {1, 2}); } public void testReset() throws Exception { WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("abcde")); - NGramTokenFilter filter = new NGramTokenFilter(tokenizer, 1, 1); - assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}); + NGramTokenFilter filter = new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 1, 1); + assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, new int[]{1,0,0,0,0}); tokenizer.setReader(new StringReader("abcde")); - assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}); + assertTokenStreamContents(filter, new String[]{"a","b","c","d","e"}, new int[]{0,1,2,3,4}, new int[]{1,2,3,4,5}, new int[]{1,0,0,0,0}); } // LUCENE-3642 @@ -112,14 +114,15 @@ protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); TokenFilter filters = new ASCIIFoldingFilter(tokenizer); - filters = new NGramTokenFilter(filters, 2, 2); + filters = new NGramTokenFilter(TEST_VERSION_CURRENT, filters, 2, 2); return new TokenStreamComponents(tokenizer, filters); } }; assertAnalyzesTo(analyzer, "mosfellsbær", new String[] { "mo", "os", "sf", "fe", "el", "ll", "ls", "sb", "ba", "ae", "er" }, new int[] { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }, - new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }); + new int[] { 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11 }, + new int[] { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }); } /** blast some random strings through the analyzer */ @@ -129,7 +132,7 @@ protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); return new TokenStreamComponents(tokenizer, - new NGramTokenFilter(tokenizer, 2, 4)); + new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 2, 4)); } }; checkRandomData(random(), a, 1000*RANDOM_MULTIPLIER, 20, false, false); @@ -142,7 +145,7 @@ protected TokenStreamComponents createComponents(String fieldName, Reader reader) { Tokenizer tokenizer = new KeywordTokenizer(reader); return new TokenStreamComponents(tokenizer, - new NGramTokenFilter(tokenizer, 2, 15)); + new NGramTokenFilter(TEST_VERSION_CURRENT, tokenizer, 2, 15)); } }; checkAnalysisConsistency(random, a, random.nextBoolean(), "");