Index: lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== --- lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (revision 944701) +++ lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (working copy) @@ -128,7 +128,7 @@ /** Filters MockTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true)); + return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)); } } @@ -158,7 +158,7 @@ public QueryParser getParser(Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", a); qp.setDefaultOperator(QueryParser.OR_OPERATOR); return qp; @@ -228,7 +228,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", a); qp.setDefaultOperator(QueryParser.AND_OPERATOR); return qp.parse(query); @@ -456,7 +456,7 @@ assertQueryEquals("[ a TO z]", null, "[a TO z]"); assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod()); - QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockAnalyzer.SIMPLE, true)); + QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockTokenizer.SIMPLE, true)); qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod()); @@ -579,7 +579,7 @@ final String defaultField = "default"; final String monthField = "month"; final String hourField = "hour"; - QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockAnalyzer.SIMPLE, true)); + QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "field", new MockAnalyzer(MockTokenizer.SIMPLE, true)); // Don't set any date resolution and verify if DateField is used assertDateRangeQueryEquals(qp, defaultField, startDate, endDate, Index: lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 944701) +++ lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -1,5 +1,13 @@ package org.apache.lucene.analysis; +import java.util.Arrays; + +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.BasicAutomata; +import org.apache.lucene.util.automaton.BasicOperations; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -19,6 +27,7 @@ public class TestMockAnalyzer extends BaseTokenStreamTestCase { + /** Test a configuration that behaves a lot like WhitespaceAnalyzer */ public void testWhitespace() throws Exception { Analyzer a = new MockAnalyzer(); assertAnalyzesTo(a, "A bc defg hiJklmn opqrstuv wxy z ", @@ -29,8 +38,9 @@ new String[] { "break", "on", "whitespace" }); } + /** Test a configuration that behaves a lot like SimpleAnalyzer */ public void testSimple() throws Exception { - Analyzer a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true); assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ", new String[] { "a", "bc", "defg", "hijklmn", "opqrstuv", "wxy", "z" }); assertAnalyzesToReuse(a, "aba4cadaba-Shazam", @@ -39,8 +49,9 @@ new String[] { "break", "on", "letters" }); } + /** Test a configuration that behaves a lot like KeywordAnalyzer */ public void testKeyword() throws Exception { - Analyzer a = new MockAnalyzer(MockAnalyzer.KEYWORD, false); + Analyzer a = new MockAnalyzer(MockTokenizer.KEYWORD, false); assertAnalyzesTo(a, "a-bc123 defg+hijklmn567opqrstuv78wxy_z ", new String[] { "a-bc123 defg+hijklmn567opqrstuv78wxy_z " }); assertAnalyzesToReuse(a, "aba4cadaba-Shazam", @@ -48,4 +59,40 @@ assertAnalyzesToReuse(a, "break+on/Nothing", new String[] { "break+on/Nothing" }); } + + /** Test a configuration that behaves a lot like StopAnalyzer */ + public void testStop() throws Exception { + Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true); + assertAnalyzesTo(a, "the quick brown a fox", + new String[] { "quick", "brown", "fox" }, + new int[] { 2, 1, 2 }); + + // disable positions + a = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, false); + assertAnalyzesTo(a, "the quick brown a fox", + new String[] { "quick", "brown", "fox" }, + new int[] { 1, 1, 1 }); + } + + /** Test a configuration that behaves a lot like KeepWordFilter */ + public void testKeep() throws Exception { + CharacterRunAutomaton keepWords = + new CharacterRunAutomaton( + BasicOperations.complement( + Automaton.union( + Arrays.asList(BasicAutomata.makeString("foo"), BasicAutomata.makeString("bar"))))); + Analyzer a = new MockAnalyzer(MockTokenizer.SIMPLE, true, keepWords, true); + assertAnalyzesTo(a, "quick foo brown bar bar fox foo", + new String[] { "foo", "bar", "bar", "foo" }, + new int[] { 2, 2, 1, 2 }); + } + + /** Test a configuration that behaves a lot like LengthFilter */ + public void testLength() throws Exception { + CharacterRunAutomaton length5 = new CharacterRunAutomaton(new RegExp(".{5,}").toAutomaton()); + Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, true, length5, true); + assertAnalyzesTo(a, "ok toolong fine notfine", + new String[] { "ok", "fine" }, + new int[] { 1, 2 }); + } } Index: lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java (revision 944701) +++ lucene/src/test/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -21,52 +21,55 @@ import java.io.Reader; import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.RegExp; /** * Analyzer for testing */ -public final class MockAnalyzer extends Analyzer { - /** Acts Similar to WhitespaceAnalyzer */ - public static final CharacterRunAutomaton WHITESPACE = - new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton()); - /** Acts Similar to KeywordAnalyzer. - * TODO: Keyword returns an "empty" token for an empty reader... - */ - public static final CharacterRunAutomaton KEYWORD = - new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); - /** Acts like SimpleAnalyzer/LetterTokenizer. */ - // the ugly regex below is Unicode 5.2 [:Letter:] - public static final CharacterRunAutomaton SIMPLE = - new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); - +public final class MockAnalyzer extends Analyzer { private final CharacterRunAutomaton runAutomaton; private final boolean lowerCase; - - public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) { + private final CharacterRunAutomaton filter; + private final boolean enablePositionIncrements; + + public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) { this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; + this.filter = filter; + this.enablePositionIncrements = enablePositionIncrements; } + + public MockAnalyzer(CharacterRunAutomaton runAutomaton, boolean lowerCase) { + this(runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false); + } public MockAnalyzer() { - this(WHITESPACE, true); + this(MockTokenizer.WHITESPACE, true); } @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new MockTokenizer(reader, runAutomaton, lowerCase); + MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + return new MockTokenFilter(tokenizer, filter, enablePositionIncrements); } + private class SavedStreams { + MockTokenizer tokenizer; + MockTokenFilter filter; + } + @Override public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { - MockTokenizer t = (MockTokenizer) getPreviousTokenStream(); - if (t == null) { - t = new MockTokenizer(reader, runAutomaton, lowerCase); - setPreviousTokenStream(t); + SavedStreams saved = (SavedStreams) getPreviousTokenStream(); + if (saved == null) { + saved = new SavedStreams(); + saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements); + setPreviousTokenStream(saved); + return saved.filter; } else { - t.reset(reader); + saved.tokenizer.reset(reader); + return saved.filter; } - return t; } } \ No newline at end of file Index: lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java (revision 944701) +++ lucene/src/test/org/apache/lucene/analysis/MockTokenizer.java (working copy) @@ -22,11 +22,25 @@ import org.apache.lucene.util.Version; import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; /** * Automaton-based tokenizer for testing. Optionally lowercases. */ public class MockTokenizer extends CharTokenizer { + /** Acts Similar to WhitespaceTokenizer */ + public static final CharacterRunAutomaton WHITESPACE = + new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton()); + /** Acts Similar to KeywordTokenizer. + * TODO: Keyword returns an "empty" token for an empty reader... + */ + public static final CharacterRunAutomaton KEYWORD = + new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); + /** Acts like LetterTokenizer. */ + // the ugly regex below is Unicode 5.2 [:Letter:] + public static final CharacterRunAutomaton SIMPLE = + new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); + private final CharacterRunAutomaton runAutomaton; private final boolean lowerCase; private int state; Index: lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/TestFuzzyQuery2.java (working copy) @@ -22,6 +22,7 @@ import java.io.InputStreamReader; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; @@ -77,7 +78,7 @@ int terms = (int) Math.pow(2, bits); RAMDirectory dir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(MockAnalyzer.KEYWORD, false), + IndexWriter writer = new IndexWriter(dir, new MockAnalyzer(MockTokenizer.KEYWORD, false), IndexWriter.MaxFieldLength.UNLIMITED); Document doc = new Document(); Index: lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (working copy) @@ -334,7 +334,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.WHITESPACE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); return new PayloadFilter(result, fieldName); } } Index: lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.document.Document; @@ -617,7 +618,7 @@ /* build an index */ RAMDirectory farsiIndex = new RAMDirectory(); IndexWriter writer = new IndexWriter(farsiIndex, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))); + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))); Document doc = new Document(); doc.add(new Field("content", "\u0633\u0627\u0628", Field.Store.YES, Field.Index.NOT_ANALYZED)); @@ -657,7 +658,7 @@ /* build an index */ RAMDirectory danishIndex = new RAMDirectory(); IndexWriter writer = new IndexWriter(danishIndex, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))); + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))); // Danish collation orders the words below in the given order // (example taken from TestSort.testInternationalSort() ). Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy) @@ -20,7 +20,6 @@ import java.util.Collection; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; @@ -59,7 +58,7 @@ private class PayloadAnalyzer extends Analyzer { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); result = new PayloadFilter(result, fieldName); return result; } Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (working copy) @@ -30,10 +30,8 @@ import org.apache.lucene.search.spans.Spans; import org.apache.lucene.search.spans.TermSpans; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.LowerCaseTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.index.IndexWriterConfig; @@ -69,7 +67,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); result = new PayloadFilter(result, fieldName); return result; } Index: lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java (working copy) @@ -52,7 +52,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); result = new PayloadFilter(result, fieldName); return result; } Index: lucene/src/test/org/apache/lucene/search/spans/TestBasics.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/spans/TestBasics.java (working copy) @@ -20,7 +20,7 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; @@ -58,7 +58,7 @@ super.setUp(); RAMDirectory directory = new RAMDirectory(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))); + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))); //writer.infoStream = System.out; for (int i = 0; i < 1000; i++) { Document doc = new Document(); Index: lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java (working copy) @@ -467,7 +467,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); result = new PayloadFilter(result, fieldName); return result; } @@ -519,7 +519,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new MockTokenizer(reader, MockAnalyzer.SIMPLE, true); + TokenStream result = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); result = new PayloadFilter(result, fieldName); return result; } Index: lucene/src/test/org/apache/lucene/search/TestTermVectors.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestTermVectors.java (revision 944701) +++ lucene/src/test/org/apache/lucene/search/TestTermVectors.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -44,7 +45,7 @@ protected void setUp() throws Exception { super.setUp(); IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))); + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))); //writer.setUseCompoundFile(true); //writer.infoStream = System.out; for (int i = 0; i < 1000; i++) { @@ -96,7 +97,7 @@ public void testTermVectorsFieldOrder() throws IOException { Directory dir = new MockRAMDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true))); + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true))); Document doc = new Document(); doc.add(new Field("c", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); doc.add(new Field("a", "some content here", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); @@ -236,7 +237,7 @@ try { IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(MockAnalyzer.SIMPLE, true)) + new MockAnalyzer(MockTokenizer.SIMPLE, true)) .setOpenMode(OpenMode.CREATE)); writer.addDocument(testDoc1); writer.addDocument(testDoc2); @@ -352,7 +353,7 @@ // Test only a few docs having vectors public void testRareVectors() throws IOException { IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(MockAnalyzer.SIMPLE, true)) + TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true)) .setOpenMode(OpenMode.CREATE)); for (int i = 0; i < 100; i++) { Document doc = new Document(); @@ -386,7 +387,7 @@ public void testMixedVectrosVectors() throws IOException { IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(MockAnalyzer.SIMPLE, true)).setOpenMode(OpenMode.CREATE)); + new MockAnalyzer(MockTokenizer.SIMPLE, true)).setOpenMode(OpenMode.CREATE)); Document doc = new Document(); doc.add(new Field("field", "one", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.NO)); Index: lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 944701) +++ lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy) @@ -101,7 +101,7 @@ private static class PayloadAnalyzer extends Analyzer { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new PayloadFilter(new MockTokenizer(reader, MockAnalyzer.WHITESPACE, true)); + return new PayloadFilter(new MockTokenizer(reader, MockTokenizer.WHITESPACE, true)); } } Index: lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java (revision 944701) +++ lucene/src/test/org/apache/lucene/index/TestThreadedOptimize.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.MockRAMDirectory; @@ -35,7 +36,7 @@ public class TestThreadedOptimize extends LuceneTestCase { - private static final Analyzer ANALYZER = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + private static final Analyzer ANALYZER = new MockAnalyzer(MockTokenizer.SIMPLE, true); private final static int NUM_THREADS = 3; //private final static int NUM_THREADS = 5; Index: lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java =================================================================== --- lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (revision 944680) +++ lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java (working copy) @@ -144,7 +144,7 @@ /** Filters MockTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true)); + return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)); } } @@ -204,7 +204,7 @@ public StandardQueryParser getParser(Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer(a); @@ -294,7 +294,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); StandardQueryParser qp = new StandardQueryParser(); qp.setAnalyzer(a); qp.setDefaultOperator(Operator.AND); Index: lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java =================================================================== --- lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (revision 944680) +++ lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java (working copy) @@ -141,7 +141,7 @@ /** Filters MockTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true)); + return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)); } } @@ -219,7 +219,7 @@ public QueryParserWrapper getParser(Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); QueryParserWrapper qp = new QueryParserWrapper("field", a); qp.setDefaultOperator(QueryParserWrapper.OR_OPERATOR); return qp; @@ -304,7 +304,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); QueryParserWrapper qp = new QueryParserWrapper("field", a); qp.setDefaultOperator(QueryParserWrapper.AND_OPERATOR); return qp.parse(query); @@ -554,7 +554,7 @@ assertEquals(MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT, ((TermRangeQuery)getQuery("[ a TO z]", null)).getRewriteMethod()); QueryParserWrapper qp = new QueryParserWrapper("field", - new MockAnalyzer(MockAnalyzer.SIMPLE, true)); + new MockAnalyzer(MockTokenizer.SIMPLE, true)); qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE); assertEquals(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE,((TermRangeQuery)qp.parse("[ a TO z]")).getRewriteMethod()); @@ -685,7 +685,7 @@ final String monthField = "month"; final String hourField = "hour"; QueryParserWrapper qp = new QueryParserWrapper("field", - new MockAnalyzer(MockAnalyzer.SIMPLE, true)); + new MockAnalyzer(MockTokenizer.SIMPLE, true)); // Don't set any date resolution and verify if DateField is used assertDateRangeQueryEquals(qp, defaultField, startDate, endDate, Index: lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (revision 944680) +++ lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/ext/TestExtendableQueryParser.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.queryParser.ParseException; import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.queryParser.TestQueryParser; @@ -46,7 +47,7 @@ public QueryParser getParser(Analyzer a, Extensions extensions) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); QueryParser qp = extensions == null ? new ExtendableQueryParser( TEST_VERSION_CURRENT, "field", a) : new ExtendableQueryParser( TEST_VERSION_CURRENT, "field", a, extensions); Index: lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (revision 944680) +++ lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java (working copy) @@ -20,7 +20,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceAnalyzer; @@ -100,7 +99,7 @@ /** Filters MockTokenizer with StopFilter. */ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { - return new QPTestFilter(new MockTokenizer(reader, MockAnalyzer.SIMPLE, true)); + return new QPTestFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)); } } @@ -130,7 +129,7 @@ public PrecedenceQueryParser getParser(Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a); qp.setDefaultOperator(PrecedenceQueryParser.OR_OPERATOR); return qp; @@ -175,7 +174,7 @@ public Query getQueryDOA(String query, Analyzer a) throws Exception { if (a == null) - a = new MockAnalyzer(MockAnalyzer.SIMPLE, true); + a = new MockAnalyzer(MockTokenizer.SIMPLE, true); PrecedenceQueryParser qp = new PrecedenceQueryParser("field", a); qp.setDefaultOperator(PrecedenceQueryParser.AND_OPERATOR); return qp.parse(query);