Index: modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (revision 1292616) +++ modules/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceCharFilter.java (working copy) @@ -21,6 +21,7 @@ import java.io.Reader; import java.io.StringReader; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; @@ -29,6 +30,7 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util._TestUtil; /** * Tests {@link PatternReplaceCharFilter} @@ -178,18 +180,33 @@ /** blast some random strings through the analyzer */ public void testRandomStrings() throws Exception { - Analyzer a = new Analyzer() { - @Override - protected TokenStreamComponents createComponents(String fieldName, Reader reader) { - Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); - return new TokenStreamComponents(tokenizer, tokenizer); - } + int numPatterns = atLeast(100); + for (int i = 0; i < numPatterns; i++) { + final Pattern p = randomPattern(); + final String replacement = _TestUtil.randomSimpleString(random); + Analyzer a = new Analyzer() { + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); + return new TokenStreamComponents(tokenizer, tokenizer); + } - @Override - protected Reader initReader(Reader reader) { - return new PatternReplaceCharFilter(Pattern.compile("a"), "b", CharReader.get(reader)); + @Override + protected Reader initReader(Reader reader) { + return new PatternReplaceCharFilter(p, replacement, CharReader.get(reader)); + } + }; + checkRandomData(random, a, 1000*RANDOM_MULTIPLIER, true); // only ascii + } + } + + public static Pattern randomPattern() { + while (true) { + try { + return Pattern.compile(_TestUtil.randomRegexpishString(random)); + } catch (PatternSyntaxException ignored) { + // if at first you don't succeed... } - }; - checkRandomData(random, a, 10000*RANDOM_MULTIPLIER); + } } -} + } Index: lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1292616) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -246,15 +246,22 @@ assertAnalyzesToReuse(a, input, new String[]{expected}); } - // simple utility method for blasting tokenstreams with data to make sure they don't do anything crazy - // TODO: add a MockCharStream, and use it here too, to ensure that correctOffset etc is being done by tokenizers. + /** utility method for blasting tokenstreams with data to make sure they don't do anything crazy */ public static void checkRandomData(Random random, Analyzer a, int iterations) throws IOException { - checkRandomData(random, a, iterations, 20); + checkRandomData(random, a, iterations, false); + } + + /** + * utility method for blasting tokenstreams with data to make sure they don't do anything crazy + * @param simple true if only ascii strings will be used (try to avoid) + */ + public static void checkRandomData(Random random, Analyzer a, int iterations, boolean simple) throws IOException { + checkRandomData(random, a, iterations, 20, simple); // now test with multiple threads int numThreads = _TestUtil.nextInt(random, 4, 8); Thread threads[] = new Thread[numThreads]; for (int i = 0; i < threads.length; i++) { - threads[i] = new AnalysisThread(new Random(random.nextLong()), a, iterations); + threads[i] = new AnalysisThread(new Random(random.nextLong()), a, iterations, simple); } for (int i = 0; i < threads.length; i++) { threads[i].start(); @@ -272,11 +279,13 @@ final int iterations; final Random random; final Analyzer a; + final boolean simple; - AnalysisThread(Random random, Analyzer a, int iterations) { + AnalysisThread(Random random, Analyzer a, int iterations, boolean simple) { this.random = random; this.a = a; this.iterations = iterations; + this.simple = simple; } @Override @@ -284,32 +293,36 @@ try { // see the part in checkRandomData where it replays the same text again // to verify reproducability/reuse: hopefully this would catch thread hazards. - checkRandomData(random, a, iterations, 20); + checkRandomData(random, a, iterations, 20, simple); } catch (IOException e) { throw new RuntimeException(e); } } }; - public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException { - checkRandomData(random, a, iterations, maxWordLength, random.nextBoolean()); + public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean simple) throws IOException { + checkRandomData(random, a, iterations, maxWordLength, random.nextBoolean(), simple); } - public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter) throws IOException { + public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength, boolean useCharFilter, boolean simple) throws IOException { for (int i = 0; i < iterations; i++) { String text; - switch(_TestUtil.nextInt(random, 0, 4)) { - case 0: - text = _TestUtil.randomSimpleString(random); - break; - case 1: - text = _TestUtil.randomRealisticUnicodeString(random, maxWordLength); - break; - case 2: - text = _TestUtil.randomHtmlishString(random, maxWordLength); - break; - default: - text = _TestUtil.randomUnicodeString(random, maxWordLength); + if (simple) { + text = random.nextBoolean() ? _TestUtil.randomSimpleString(random) : _TestUtil.randomHtmlishString(random, maxWordLength); + } else { + switch(_TestUtil.nextInt(random, 0, 4)) { + case 0: + text = _TestUtil.randomSimpleString(random); + break; + case 1: + text = _TestUtil.randomRealisticUnicodeString(random, maxWordLength); + break; + case 2: + text = _TestUtil.randomHtmlishString(random, maxWordLength); + break; + default: + text = _TestUtil.randomUnicodeString(random, maxWordLength); + } } if (VERBOSE) { Index: lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (revision 1292616) +++ lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (working copy) @@ -249,6 +249,36 @@ } } + /** + * Returns a String thats "regexpish" (contains lots of operators typically found in regular expressions) + * If you call this enough times, you might get a valid regex! + */ + public static String randomRegexpishString(Random r) { + final int end = r.nextInt(20); + if (end == 0) { + // allow 0 length + return ""; + } + final char[] buffer = new char[end]; + for (int i = 0; i < end; i++) { + int t = r.nextInt(11); + if (t == 0) { + buffer[i] = (char) _TestUtil.nextInt(r, 97, 102); + } + else if (1 == t) buffer[i] = '.'; + else if (2 == t) buffer[i] = '?'; + else if (3 == t) buffer[i] = '*'; + else if (4 == t) buffer[i] = '+'; + else if (5 == t) buffer[i] = '('; + else if (6 == t) buffer[i] = ')'; + else if (7 == t) buffer[i] = '-'; + else if (8 == t) buffer[i] = '['; + else if (9 == t) buffer[i] = ']'; + else if (10 == t) buffer[i] = '|'; + } + return new String(buffer, 0, end); + } + private static final String[] HTML_CHAR_ENTITIES = { "AElig", "Aacute", "Acirc", "Agrave", "Alpha", "AMP", "Aring", "Atilde", "Auml", "Beta", "COPY", "Ccedil", "Chi", "Dagger", "Delta", "ETH",