Index: lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java =================================================================== --- lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (revision 1449616) +++ lucene/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (working copy) @@ -22,14 +22,20 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.collation.tokenattributes.ICUCollatedTermAttributeImpl; +import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.BytesRef; -import java.util.Locale; +import java.io.StringReader; +import java.util.*; public class TestICUCollationKeyAnalyzer extends CollationTestBase { private Collator collator = Collator.getInstance(new Locale("fa")); private Analyzer analyzer = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, collator); + private Analyzer appendingAnalyzer = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, collator, true); private BytesRef firstRangeBeginning = new BytesRef (collator.getCollationKey(firstRangeBeginningOriginal).toByteArray()); @@ -50,9 +56,19 @@ secondRangeBeginning, secondRangeEnd); } + public void testFarsiRangeQueryCollatingAppending() throws Exception { + testFarsiRangeQueryCollating(appendingAnalyzer, firstRangeBeginning, firstRangeEnd, + secondRangeBeginning, secondRangeEnd); + } + public void testFarsiTermRangeQuery() throws Exception { + testFarsiTermRangeQuery(analyzer, firstRangeBeginning, firstRangeEnd, + secondRangeBeginning, secondRangeEnd); + } + + public void testFarsiTermRangeQueryAppending() throws Exception { testFarsiTermRangeQuery - (analyzer, firstRangeBeginning, firstRangeEnd, + (appendingAnalyzer, firstRangeBeginning, firstRangeEnd, secondRangeBeginning, secondRangeEnd); } @@ -61,7 +77,7 @@ // // Copied (and slightly modified) from // org.apache.lucene.search.TestSort.testInternationalSort() - // + // public void testCollationKeySort() throws Exception { Analyzer usAnalyzer = new ICUCollationKeyAnalyzer (TEST_VERSION_CURRENT, Collator.getInstance(Locale.ROOT)); @@ -78,7 +94,30 @@ (usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer, "BFJHD", "ECAGI", "BJDFH", "BJDHF"); } - + + public void testCollationKeySortAppend() throws Exception { + testCollationKeySortAppend(true); + } + + // TODO: Make testCollationKeySort call this method with false + // Optionally appends the originating terms to the collation keys + public void testCollationKeySortAppend(boolean appendTerm) throws Exception { + Analyzer usAnalyzer = new ICUCollationKeyAnalyzer + (TEST_VERSION_CURRENT, Collator.getInstance(Locale.ROOT), appendTerm); + Analyzer franceAnalyzer = new ICUCollationKeyAnalyzer + (TEST_VERSION_CURRENT, Collator.getInstance(Locale.FRANCE), appendTerm); + Analyzer swedenAnalyzer = new ICUCollationKeyAnalyzer + (TEST_VERSION_CURRENT, Collator.getInstance(new Locale("sv", "se")), appendTerm); + Analyzer denmarkAnalyzer = new ICUCollationKeyAnalyzer + (TEST_VERSION_CURRENT, Collator.getInstance(new Locale("da", "dk")), appendTerm); + + // The ICU Collator and java.text.Collator implementations differ in their + // orderings - "BFJHD" is the ordering for the ICU Collator for Locale.ROOT. + testCollationKeySort + (usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer, + "BFJHD", "ECAGI", "BJDFH", "BJDHF"); + } + public void testThreadSafe() throws Exception { int iters = 20 * RANDOM_MULTIPLIER; for (int i = 0; i < iters; i++) { @@ -88,4 +127,57 @@ assertThreadSafe(new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, collator)); } } + + public void testOriginatingTermExtraction() throws Exception { + final List locales = Arrays.asList( + Locale.ROOT, Locale.FRANCE, new Locale("sv", "se"), new Locale("da", "dk")); + final List terms = Arrays.asList( + "foo", "bar", "p\u00E9ch\u00E9", "H\u00C5T", ""); + + List analyzers = new ArrayList + (locales.size()); + for (Locale locale: locales) { + analyzers.add(new ICUCollationKeyAnalyzer( + TEST_VERSION_CURRENT, Collator.getInstance(locale), true)); + } + for (Analyzer analyzer: analyzers) { + for (String term: terms) { + TokenStream stream = analyzer.tokenStream("dummy", new StringReader(term)); + stream.reset(); + assertTrue( + "There should be a token for term '" + term + "' with analyzer " + analyzer, + stream.incrementToken()); + Iterator ais = stream.getAttributeImplsIterator(); + +// Why does this not work? +// ICUCollatedTermAttributeImpl ai = stream.getAttribute(ICUCollatedTermAttributeImpl.class); + + boolean gotCollated = false; + while (ais.hasNext()) { + AttributeImpl ai = ais.next(); + if (ai instanceof ICUCollatedTermAttributeImpl) { + ICUCollatedTermAttributeImpl iai = (ICUCollatedTermAttributeImpl)ai; + iai.fillBytesRef(); + String rebuild = getOriginating(iai.getBytesRef()).utf8ToString(); + assertEquals("The reconstructed term should match the originating", + term, rebuild); + gotCollated = true; + } + } + assertTrue("There should have been an ICUCollatedTermAttributeImpl for term '" + term + + "' with analyzer " + analyzer, + gotCollated); + } + } + } + + private BytesRef getOriginating(BytesRef concatenated) { + for (int i = 0 ; i < concatenated.length ; i++) { + if (concatenated.bytes[concatenated.offset+i] == 0) { + return new BytesRef(concatenated.bytes, i+1, concatenated.length-i-1); + } + } + // We should not get here with proper input as "" -> [01 01 00] + return new BytesRef(); + } } Index: lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationAttributeFactory.java =================================================================== --- lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationAttributeFactory.java (revision 1449616) +++ lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationAttributeFactory.java (working copy) @@ -65,7 +65,8 @@ public class ICUCollationAttributeFactory extends AttributeSource.AttributeFactory { private final Collator collator; private final AttributeSource.AttributeFactory delegate; - + private final boolean appendTerm; + /** * Create an ICUCollationAttributeFactory, using * {@link org.apache.lucene.util.AttributeSource.AttributeFactory#DEFAULT_ATTRIBUTE_FACTORY} as the @@ -73,25 +74,52 @@ * @param collator CollationKey generator */ public ICUCollationAttributeFactory(Collator collator) { - this(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator); + this(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, false); } - + /** + * Create an ICUCollationAttributeFactory, using + * {@link org.apache.lucene.util.AttributeSource.AttributeFactory#DEFAULT_ATTRIBUTE_FACTORY} as the + * factory for all other attributes. + * @param collator CollationKey generator + * @param appendTerm if true, the originating term is appended to the + * generated collation key. The sort order for the extended + * keys is unaffected by this addition. + */ + public ICUCollationAttributeFactory(Collator collator, boolean appendTerm) { + this(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, appendTerm); + } + + /** * Create an ICUCollationAttributeFactory, using the supplied Attribute * Factory as the factory for all other attributes. * @param delegate Attribute Factory * @param collator CollationKey generator */ public ICUCollationAttributeFactory(AttributeSource.AttributeFactory delegate, Collator collator) { + this (delegate, collator, false); + } + + /** + * Create an ICUCollationAttributeFactory, using the supplied Attribute + * Factory as the factory for all other attributes. + * @param delegate Attribute Factory + * @param collator CollationKey generator + * @param appendTerm if true, the originating term is appended to the + * generated collation key. The sort order for the extended + * keys is unaffected by this addition. + */ + public ICUCollationAttributeFactory(AttributeSource.AttributeFactory delegate, Collator collator, boolean appendTerm) { this.delegate = delegate; this.collator = collator; + this.appendTerm = appendTerm; } - + @Override public AttributeImpl createAttributeInstance( Class attClass) { return attClass.isAssignableFrom(ICUCollatedTermAttributeImpl.class) - ? new ICUCollatedTermAttributeImpl(collator) + ? new ICUCollatedTermAttributeImpl(collator, appendTerm) : delegate.createAttributeInstance(attClass); } } Index: lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java =================================================================== --- lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (revision 1449616) +++ lucene/analysis/icu/src/java/org/apache/lucene/collation/ICUCollationKeyAnalyzer.java (working copy) @@ -76,10 +76,23 @@ * @param collator CollationKey generator */ public ICUCollationKeyAnalyzer(Version matchVersion, Collator collator) { - this.factory = new ICUCollationAttributeFactory(collator); + this(matchVersion, collator, false); } + /** + * Create a new ICUCollationKeyAnalyzer, using the specified collator. + * + * @param matchVersion compatibility version + * @param collator CollationKey generator + * @param appendTerm if true, the originating terms are appended to the + * generated collation keys. The sort order for the extended + * keys is unaffected by this addition. + */ + public ICUCollationKeyAnalyzer(Version matchVersion, Collator collator, boolean appendTerm) { + this.factory = new ICUCollationAttributeFactory(collator, appendTerm); + } + @Override protected TokenStreamComponents createComponents(String fieldName, Reader reader) { KeywordTokenizer tokenizer = new KeywordTokenizer(factory, reader, KeywordTokenizer.DEFAULT_BUFFER_SIZE); Index: lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java =================================================================== --- lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java (revision 1449616) +++ lucene/analysis/icu/src/java/org/apache/lucene/collation/tokenattributes/ICUCollatedTermAttributeImpl.java (working copy) @@ -23,19 +23,40 @@ import com.ibm.icu.text.Collator; import com.ibm.icu.text.RawCollationKey; +import java.io.UnsupportedEncodingException; + /** * Extension of {@link CharTermAttributeImpl} that encodes the term * text as a binary Unicode collation key instead of as UTF-8 bytes. + *

+ * Optionally, the originating BytesRef is appended to the collation key. + * As the last byte and only the last byte of a ICU collation key is 0, + * this makes it possible to retrieve the originating BytesRef from the key + * by splitting on byte 0. + *

+ * Intended usage is for faceting with collator ordered results. */ public class ICUCollatedTermAttributeImpl extends CharTermAttributeImpl { private final Collator collator; private final RawCollationKey key = new RawCollationKey(); + private final boolean appendTerm; /** * Create a new ICUCollatedTermAttributeImpl * @param collator Collation key generator */ public ICUCollatedTermAttributeImpl(Collator collator) { + this(collator, false); + } + + /** + * Create a new ICUCollatedTermAttributeImpl + * @param collator Collation key generator + * @param appendTerm if true, the bytes from the originating term are + * appended to the collation key. + */ + public ICUCollatedTermAttributeImpl(Collator collator, boolean appendTerm) { + this.appendTerm = appendTerm; // clone the collator: see http://userguide.icu-project.org/collation/architecture try { this.collator = (Collator) collator.clone(); @@ -46,11 +67,45 @@ @Override public int fillBytesRef() { - BytesRef bytes = getBytesRef(); + if (appendTerm) { + return fillBytesRefAppend(); + } + + BytesRef result = getBytesRef(); collator.getRawCollationKey(toString(), key); - bytes.bytes = key.bytes; - bytes.offset = 0; - bytes.length = key.size; - return bytes.hashCode(); + result.length = key.size; + result.bytes = key.bytes; + result.offset = 0; + return result.hashCode(); } + + /** + * @return an ICU collation key with the originating term appended as UTF-8. + */ + public int fillBytesRefAppend() { + String originating = toString(); + collator.getRawCollationKey(originating, key); + byte[] originatingUTF8; + try { + originatingUTF8 = originating.getBytes("utf-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + + BytesRef result = getBytesRef(); + int totalLength = key.size + originatingUTF8.length; + byte[] content; + if (key.bytes.length < totalLength) { // We need to expand + content = new byte[totalLength]; + System.arraycopy(key.bytes, 0, content, 0, key.size); + } else { // There is room for the originating term + content = key.bytes; + } + System.arraycopy(originatingUTF8, result.offset, + content, key.size, originatingUTF8.length); + result.bytes = content; + result.offset = 0; + result.length = totalLength; + return result.hashCode(); + } }