public void _testRandomStrings() throws IOException { Analyzer a = new ReusableAnalyzerBase() { @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { return new TokenStreamComponents(new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)); } @Override protected Reader initReader(Reader reader) { return new ICUNormalizer2CharFilter(CharReader.get(reader)); } }; checkRandomData(random, a, 10000*RANDOM_MULTIPLIER); } @Test public void testRandomStringsDebug() throws IOException { System.setProperty("tests.verbose", "true"); final int RUN = 10; int failCount = 0; for (int count = RUN ; count > 0; count --){ AssertionError e = null; String errStr = null; ByteArrayOutputStream out = new ByteArrayOutputStream(); PrintStream origOut = System.out; System.setOut(new PrintStream(out,true,"UTF-8")); ByteArrayOutputStream err = new ByteArrayOutputStream(); PrintStream origErr = System.err; System.setErr(new PrintStream(err,true,"UTF-8")); try{ _testRandomStrings(); } catch(AssertionError e2){ failCount ++; e = e2; } finally { System.setErr(origErr); System.setOut(origOut); } if(e != null){ System.out.println("---------"); System.out.println(e.getMessage()); for(StackTraceElement s: e.getStackTrace()){ if(s.getClassName().contains("org.junit")) continue; System.out.println(s.toString()); break; } errStr = err.toString("UTF-8"); analyzeAndPrintFailedInputText(errStr); System.out.println("---------"); } } if(failCount > 0){ fail(String.format("%d AssertionErrors in %d runs.", failCount ,RUN)); } } private void analyzeAndPrintFailedInputText(String errStr) throws IOException { Matcher matcher = Pattern.compile("text='(.*)'\n", Pattern.DOTALL).matcher(errStr); matcher.find(); String failedInputText = matcher.group(1); printListOfChars(failedInputText); Normalizer2 normalizer = Normalizer2.getInstance(null, "nfkc_cf", Normalizer2.Mode.COMPOSE); String failedInputTextNormalized = normalizer.normalize(failedInputText); printListOfChars(failedInputTextNormalized); System.out.println("---------"); ICUNormalizer2CharFilter cf = new ICUNormalizer2CharFilter(CharReader.get(new StringReader(failedInputText))); while(cf.read() != -1); printOffsetCorrectMap(cf); } private void printListOfChars(String failedInputText) { System.out.println("---------"); System.out.println(failedInputText); System.out.println("---------"); char[] errStrChars = failedInputText.toCharArray(); for(int i = 0; i < failedInputText.length(); ++i){ System.out.print(i); System.out.print('\t'); boolean surrogate = false; int cp = -1; if(UCharacter.isHighSurrogate(errStrChars[i])){ if(!UCharacter.isLowSurrogate(errStrChars[i+1])){ System.out.println("HighSurrogate"); } else { surrogate = true; cp = UCharacter.codePointAt(failedInputText.toCharArray(), i); ++i; } }else if(UCharacter.isLowSurrogate(errStrChars[i])){ System.out.println("LowSurrogate"); }else { cp = UCharacter.codePointAt(failedInputText.toCharArray(), i); } System.out.print("'"); System.out.print(failedInputText.charAt(i)); System.out.print("'\t"); System.out.printf("%06X", cp); System.out.print("\t"); System.out.print(UCharacter.getAge(cp)); if(!UCharacter.isDefined(cp)) { System.out.print("\t"); System.out.print("undefined"); }else { System.out.print("\t"); System.out.print(UCharacter.getExtendedName(cp)); if(UCharacter.getName(cp) == null) { System.out.print(" (extended name)"); } } if(surrogate) { System.out.print("\t"); System.out.print("surrogate"); } System.out.println(); } } private void printOffsetCorrectMap(BaseCharFilter cf) { try { Field offsetsField = BaseCharFilter.class.getDeclaredField("offsets"); offsetsField.setAccessible(true); int[] offsets = (int[]) offsetsField.get(cf); Field diffsField = BaseCharFilter.class.getDeclaredField("diffs"); diffsField.setAccessible(true); int[] diffs = (int[]) diffsField.get(cf); System.out.println("---------"); if (diffs==null) return; for(int i = 0; i < diffs.length; ++i){ if (offsets[i] == 0 && diffs[i] == 0 ) break; System.out.print(offsets[i]); System.out.print(','); System.out.print(diffs[i]); System.out.println(); } } catch (Exception e) { throw new RuntimeException(e); } }