Index: modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java (revision 1090955) +++ modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java (working copy) @@ -258,7 +258,7 @@ assertEquals(expectedResult, buff.toString()); } - private String randomString() { + protected String randomString() { // ideally we could do this! // return _TestUtil.randomUnicodeString(random); // Index: modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (revision 1090955) +++ modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (working copy) @@ -19,11 +19,20 @@ import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.search.*; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util._TestUtil; +import java.io.IOException; import java.text.Collator; -import java.util.Locale; +import java.util.*; public class TestCollationKeyAnalyzer extends CollationTestBase { @@ -93,4 +102,103 @@ assertThreadSafe(new CollationKeyAnalyzer(TEST_VERSION_CURRENT, collator)); } } + + public void testRandomizedCollationKeySort() throws IOException { + Locale locale = randomLocale(random); + Collator collator = Collator.getInstance(locale); + int[] strengths = { Collator.IDENTICAL, Collator.PRIMARY, + Collator.SECONDARY, Collator.TERTIARY }; + collator.setStrength(strengths[random.nextInt(strengths.length)]); + int[] decompositions = { Collator.NO_DECOMPOSITION, + Collator.CANONICAL_DECOMPOSITION, + Collator.FULL_DECOMPOSITION }; + collator.setDecomposition + (decompositions[random.nextInt(decompositions.length)]); + Directory directory = newDirectory(); + Analyzer analyzer = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, collator); + RandomIndexWriter writer = new RandomIndexWriter + (random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer)); + List originalStrings = new ArrayList(); + int numDocs = 1000 * RANDOM_MULTIPLIER; + for (int docNum = 0 ; docNum < numDocs ; ++docNum) { + Document doc = new Document(); + String str = _TestUtil.randomUnicodeString(random); + originalStrings.add(str); + doc.add(new Field("contents", str, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS)); + doc.add(new Field("collation-key", str, Field.Store.NO, Field.Index.ANALYZED_NO_NORMS)); + writer.addDocument(doc); + } + IndexReader reader = writer.getReader(); + writer.close(); + + String[] sortedStrings + = originalStrings.toArray(new String[originalStrings.size()]); + Arrays.sort(sortedStrings, new CollatedCodePointComparator(collator)); + + IndexSearcher searcher = newSearcher(reader); + TopFieldDocs indexedDocs = searcher.search + (new MatchAllDocsQuery(), numDocs, + new Sort(new SortField("collation-key", SortField.STRING), + new SortField("contents", SortField.STRING))); + for (int n = 0 ; n < sortedStrings.length ; ++n) { + String sortedString = sortedStrings[n]; + Document indexedDoc + = reader.document(indexedDocs.scoreDocs[n].doc); + String indexedString = indexedDoc.get("contents"); + if (0 != UnicodeUtil.compareCodePoints(sortedString, indexedString)) { + // Don't build the error message unless there is an error + StringBuilder message = new StringBuilder(); + for (int m = Math.max(0, n - 1) ; + m <= Math.min(n + 1, sortedStrings.length) ; + ++m ) { + message.append("-----------").append("\n"); + message.append("Indexed string #").append(m).append(": "); + message.append(Arrays.toString(_TestUtil.getCodePoints + (reader.document(indexedDocs.scoreDocs[m].doc).get("contents")))); + message.append("\n"); + message.append(" Sorted string #").append(m).append(": "); + message.append + (Arrays.toString(_TestUtil.getCodePoints(sortedStrings[m]))); + message.append("\n"); + } + String[] strengthStrings + = { "IDENTICAL", "PRIMARY", "SECONDARY", "TERTIARY" }; + String[] decompositionStrings + = { "NO_DECOMPOSITION", "CANONICAL_DECOMPOSITION", + "FULL_DECOMPOSITION" }; + message.append("\nCollator strength: " + + strengthStrings[collator.getStrength()] + + " Collator decomposition: " + + decompositionStrings[collator.getDecomposition()] + + "\n\n"); + assertEquals(message.toString(), 0, + UnicodeUtil.compareCodePoints(sortedString, indexedString)); + } + } + reader.close(); + directory.close(); + } + + + /** + * Sort with Collator.compare, if thats equal, compare codepoints + */ + public class CollatedCodePointComparator implements Comparator { + + private Collator collator; + + public CollatedCodePointComparator(Collator collator) { + this.collator = collator; + } + + public int compare(String str1, String str2) { + if (null == str1 || null == str2) { + return null == str1 ? (null == str2 ? 0 : -1) : 1; + } + int comparison = collator.compare(str1, str2); + return comparison != 0 + ? comparison + : UnicodeUtil.compareCodePoints(str1, str2); + } + } } Index: lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java =================================================================== --- lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java (revision 1090955) +++ lucene/src/test/org/apache/lucene/util/TestUnicodeUtil.java (working copy) @@ -1,5 +1,9 @@ package org.apache.lucene.util; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -197,4 +201,64 @@ assertTrue(rc == -1); } } + + public void testCompareCodePoints() { + assertEquals(0, UnicodeUtil.compareCodePoints(null, null)); + assertEquals(0, UnicodeUtil.compareCodePoints("", "")); + assertTrue(UnicodeUtil.compareCodePoints(null, "") < 0); + assertTrue(UnicodeUtil.compareCodePoints("", null) > 0); + + String randomString = _TestUtil.randomUnicodeString(random); + assertEquals(0, UnicodeUtil.compareCodePoints(randomString, randomString)); + assertTrue(UnicodeUtil.compareCodePoints(null, randomString) < 0); + assertTrue(UnicodeUtil.compareCodePoints(randomString, null) > 0); + if (randomString.length() > 0) { + assertTrue(UnicodeUtil.compareCodePoints(randomString, "") > 0); + assertTrue(UnicodeUtil.compareCodePoints("", randomString) < 0); + } + @SuppressWarnings({"RedundantStringConstructorCall"}) + String randomStringCopy = new String(randomString); + assertEquals(0, UnicodeUtil.compareCodePoints(randomString, randomStringCopy)); + assertEquals(0, UnicodeUtil.compareCodePoints(randomStringCopy, randomString)); + + String randomStringPlusA = randomString + "A"; + assertTrue(UnicodeUtil.compareCodePoints(randomString, randomStringPlusA) < 0); + assertTrue(UnicodeUtil.compareCodePoints(randomStringPlusA, randomString) > 0); + + String randomStringPlusB = randomString + "B"; + assertTrue(UnicodeUtil.compareCodePoints(randomStringPlusA, randomStringPlusB) < 0); + assertTrue(UnicodeUtil.compareCodePoints(randomStringPlusB, randomStringPlusA) > 0); + + int[] randomCodePoints1 = new int[random.nextInt(20)]; + for (int i = 0 ; i < randomCodePoints1.length ; ++i) { + randomCodePoints1[i] = _TestUtil.randomCodePoint(random, true); + } + int[] randomCodePoints2 = new int[random.nextInt(20)]; + for (int i = 0 ; i < randomCodePoints2.length ; ++i) { + randomCodePoints2[i] = _TestUtil.randomCodePoint(random, true); + } + int expectedComparison = 0; + for (int i = 0 ; i < Math.min(randomCodePoints1.length, randomCodePoints2.length) ; ++i) { + if (randomCodePoints1[i] != randomCodePoints2[i]) { + expectedComparison = randomCodePoints1[i] - randomCodePoints2[i]; + expectedComparison /= Math.abs(expectedComparison); + break; + } + } + if (0 == expectedComparison) { + expectedComparison = randomCodePoints1.length - randomCodePoints2.length; + if (0 != expectedComparison) { + expectedComparison /= Math.abs(expectedComparison); + } + } + String str1 = UnicodeUtil.newString(randomCodePoints1, 0, randomCodePoints1.length); + String str2 = UnicodeUtil.newString(randomCodePoints2, 0, randomCodePoints2.length); + int actualComparison = UnicodeUtil.compareCodePoints(str1, str2); + if (0 != actualComparison) { + actualComparison /= Math.abs(actualComparison); + } + assertEquals( "str1: " + Arrays.toString(randomCodePoints1) + + "\nstr2: " + Arrays.toString(randomCodePoints2), + expectedComparison, actualComparison); + } } Index: lucene/src/java/org/apache/lucene/util/UnicodeUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (revision 1090955) +++ lucene/src/java/org/apache/lucene/util/UnicodeUtil.java (working copy) @@ -705,4 +705,22 @@ } return sb.toString(); } + + public static int compareCodePoints(String str1, String str2) { + if (null == str1 || null == str2) { + return null == str1 ? (null == str2 ? 0 : -1) : 1; + } + int index1 = 0; + int index2 = 0; + while (index1 < str1.length() && index2 < str2.length()) { + int codePoint1 = str1.codePointAt(index1); + int codePoint2 = str2.codePointAt(index2); + if (codePoint1 != codePoint2) { + return codePoint1 - codePoint2; + } + index1 += Character.charCount(codePoint1); + index2 += Character.charCount(codePoint2); + } + return str1.length() - str2.length(); + } } Index: lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (revision 1090955) +++ lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (working copy) @@ -189,23 +189,30 @@ } final char[] buffer = new char[end]; for (int i = 0; i < end; i++) { - int t = r.nextInt(5); - - if (0 == t && i < end - 1) { - // Make a surrogate pair - // High surrogate - buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff); - // Low surrogate - buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff); + int codePoint = randomCodePoint(r, i < end - 1); + Character.toChars(codePoint, buffer, i); + if (Character.isSupplementaryCodePoint(codePoint)) { + ++i; } - else if (t <= 1) buffer[i] = (char) r.nextInt(0x80); - else if (2 == t) buffer[i] = (char) nextInt(r, 0x80, 0x800); - else if (3 == t) buffer[i] = (char) nextInt(r, 0x800, 0xd7ff); - else if (4 == t) buffer[i] = (char) nextInt(r, 0xe000, 0xffff); } return new String(buffer, 0, end); } + public static int randomCodePoint(Random r, boolean isSupplementaryAcceptable) { + int codePoint; + int t = r.nextInt(5); + + if (0 == t && isSupplementaryAcceptable) { + codePoint = Character.toCodePoint((char)nextInt(r, 0xd800, 0xdbff), + (char)nextInt(r, 0xdc00, 0xdfff)); + } + else if (t <= 1) codePoint = r.nextInt(0x80); + else if (2 == t) codePoint = nextInt(r, 0x80, 0x800); + else if (3 == t) codePoint = nextInt(r, 0x800, 0xd7ff); + else codePoint = nextInt(r, 0xe000, 0xffff); + return codePoint; + } + private static final int[] blockStarts = { 0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400, 0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800, @@ -359,4 +366,14 @@ field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); } } + + public static int[] getCodePoints(String str) { + int[] codePoints = new int[str.codePointCount(0, str.length())]; + int cpNum = 0; + int strIndex = 0; + while (strIndex < str.length()) { + strIndex += Character.charCount(codePoints[cpNum++] = str.codePointAt(strIndex)); + } + return codePoints; + } }