diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 6173ece..c9d5503 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -44,6 +44,11 @@ New Features the suggester to ignore such variations. (Robert Muir, Sudarshan Gaikaiwari, Mike McCandless) +* LUCENE-4491: AnalyzingSuggester now accepts dedicated surface forms + that can differ from the suggest key. Custom suggesters can also + extend LookupResult classes to return custom suggestions including + entity meta-data. (Simon Willnauer) + API Changes * LUCENE-4399: Deprecated AppendingCodec. Lucene's term dictionaries diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java index e9f908d..47bb16d 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java @@ -106,5 +106,10 @@ public class HighFrequencyDictionary implements Dictionary { return termsEnum.getComparator(); } } + + @Override + public BytesRef surfaceForm() { + return spare; + } } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java b/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java index 13aba48..f43deb1 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqIterator.java @@ -20,6 +20,8 @@ package org.apache.lucene.search.spell; import java.io.IOException; import java.util.Comparator; +import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; @@ -32,11 +34,19 @@ public interface TermFreqIterator extends BytesRefIterator { public long weight(); /** + * Returns the surface form of the current term. Some {@link Lookup} implementations + * like {@link AnalyzingSuggester} will use the surface form as the result key instead + * of the input key. + */ + public BytesRef surfaceForm(); + + /** * Wraps a BytesRefIterator as a TermFreqIterator, with all weights * set to 1 */ public static class TermFreqIteratorWrapper implements TermFreqIterator { private BytesRefIterator wrapped; + private BytesRef last; /** * Creates a new wrapper, wrapping the specified iterator and @@ -51,12 +61,17 @@ public interface TermFreqIterator extends BytesRefIterator { } public BytesRef next() throws IOException { - return wrapped.next(); + return last = wrapped.next(); } @Override public Comparator getComparator() { return wrapped.getComparator(); } + + @Override + public BytesRef surfaceForm() { + return last; + } } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java index d686ba3..1d7141c 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java @@ -71,5 +71,10 @@ public class BufferingTermFreqIteratorWrapper implements TermFreqIterator { return comp; } + @Override + public BytesRef surfaceForm() { + return spare; + } + } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java index c032b1f..13fb5d7 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java @@ -102,6 +102,11 @@ public class FileDictionary implements Dictionary { public Comparator getComparator() { return null; } + + @Override + public BytesRef surfaceForm() { + return spare; + } } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java index 88e4f48..1d26ff6 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java @@ -36,7 +36,7 @@ public abstract class Lookup { /** * Result of a lookup. */ - public static final class LookupResult implements Comparable { + public static class LookupResult implements Comparable { /** the key's text */ public final CharSequence key; /** the key's weight */ diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java index 3aa4b79..c5d3663 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java @@ -188,5 +188,10 @@ public class SortedTermFreqIteratorWrapper implements TermFreqIterator { scratch.length -= 8; // sep + long return tmpInput.readLong(); } + + @Override + public BytesRef surfaceForm() { + return scratch; + } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java index 9f98814..b512a0a 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java @@ -38,6 +38,7 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.InputStreamDataInput; import org.apache.lucene.store.OutputStreamDataOutput; +import org.apache.lucene.search.suggest.Lookup.LookupResult; // for javadoc import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; @@ -330,11 +331,13 @@ public class AnalyzingSuggester extends Lookup { try { ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); BytesRef surfaceForm; - while ((surfaceForm = iterator.next()) != null) { + BytesRef key; + while ((key = iterator.next()) != null) { // Analyze surface form: - TokenStream ts = indexAnalyzer.tokenStream("", new StringReader(surfaceForm.utf8ToString())); - + TokenStream ts = indexAnalyzer.tokenStream("", new StringReader(key.utf8ToString())); + surfaceForm = iterator.surfaceForm(); + // Create corresponding automaton: labels are bytes // from each analyzed token, with byte 0 used as // separator between tokens: @@ -445,7 +448,6 @@ public class AnalyzingSuggester extends Lookup { analyzed.bytes[analyzed.length] = 0; analyzed.bytes[analyzed.length+1] = (byte) dedup; analyzed.length += 2; - Util.toIntsRef(analyzed, scratchInts); //System.out.println("ADD: " + scratchInts + " -> " + cost + ": " + surface.utf8ToString()); builder.add(scratchInts, outputs.newPair(cost, BytesRef.deepCopyOf(surface))); @@ -530,9 +532,9 @@ public class AnalyzingSuggester extends Lookup { FST.Arc> scratchArc = new FST.Arc>(); List results = new ArrayList(); + final BytesRef keyScratch = new BytesRef(key); if (exactFirst) { - int count = 0; for (FSTUtil.Path> path : prefixPaths) { if (fst.findTargetArc(END_BYTE, path.fstNode, scratchArc, bytesReader) != null) { @@ -561,7 +563,7 @@ public class AnalyzingSuggester extends Lookup { } MinResult> completions[] = searcher.search(); - + // NOTE: this is rather inefficient: we enumerate // every matching "exactly the same analyzed form" // path, and then do linear scan to see if one of @@ -575,10 +577,8 @@ public class AnalyzingSuggester extends Lookup { // nodes we have and the // maxSurfaceFormsPerAnalyzedForm: for(MinResult> completion : completions) { - spare.grow(completion.output.output2.length); - UnicodeUtil.UTF8toUTF16(completion.output.output2, spare); - if (CHARSEQUENCE_COMPARATOR.compare(spare, key) == 0) { - results.add(new LookupResult(spare.toString(), decodeWeight(completion.output.output1))); + if (isExactMatch(keyScratch, completion.output.output2)) { + results.add(toLookupResult(completion, spare)); break; } } @@ -611,9 +611,7 @@ public class AnalyzingSuggester extends Lookup { // In exactFirst mode, don't accept any paths // matching the surface form since that will // create duplicate results: - spare.grow(output.output2.length); - UnicodeUtil.UTF8toUTF16(output.output2, spare); - return CHARSEQUENCE_COMPARATOR.compare(spare, key) != 0; + return !isExactMatch(keyScratch, output.output2); } } }; @@ -625,9 +623,7 @@ public class AnalyzingSuggester extends Lookup { MinResult> completions[] = searcher.search(); for(MinResult> completion : completions) { - spare.grow(completion.output.output2.length); - UnicodeUtil.UTF8toUTF16(completion.output.output2, spare); - LookupResult result = new LookupResult(spare.toString(), decodeWeight(completion.output.output1)); + LookupResult result = toLookupResult(completion, spare); //System.out.println(" result=" + result); results.add(result); } @@ -637,6 +633,20 @@ public class AnalyzingSuggester extends Lookup { throw new RuntimeException(bogus); } } + + protected boolean isExactMatch(BytesRef key, BytesRef surfaceForm) { + return key.bytesEquals(surfaceForm); + } + + /** + * Converts a completion result into a {@link LookupResult}. Custom implementation can override this + * method to apply custom conversions of surface forms into {@link LookupResult}. + */ + protected LookupResult toLookupResult(MinResult> completion, CharsRef spare) { + spare.grow(completion.output.output2.length); + UnicodeUtil.UTF8toUTF16(completion.output.output2, spare); + return new LookupResult(spare.toString(), decodeWeight(completion.output.output1)); + } /** * Returns the weight associated with an input string, @@ -647,7 +657,7 @@ public class AnalyzingSuggester extends Lookup { } /** cost -> weight */ - private static int decodeWeight(long encoded) { + protected static int decodeWeight(long encoded) { return (int)(Integer.MAX_VALUE - encoded); } diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java index 2b02ac1..8ef709a 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreq.java @@ -22,13 +22,23 @@ import org.apache.lucene.util.BytesRef; public final class TermFreq { public final BytesRef term; public final long v; + public final BytesRef surfaceForm; public TermFreq(String term, long v) { - this(new BytesRef(term), v); + this(new BytesRef(term), v, new BytesRef(term)); } public TermFreq(BytesRef term, long v) { + this(term, v, term); + } + + public TermFreq(String term, long v, String surfaceForm) { + this(new BytesRef(term), v, new BytesRef(surfaceForm)); + } + + public TermFreq(BytesRef term, long v, BytesRef surfaceForm) { this.term = term; this.v = v; + this.surfaceForm = surfaceForm; } } \ No newline at end of file diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java index 06d7301..2ab4895 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java @@ -31,6 +31,7 @@ public final class TermFreqArrayIterator implements TermFreqIterator { private final Iterator i; private TermFreq current; private final BytesRef spare = new BytesRef(); + private final BytesRef surfaceSpare = new BytesRef(); public TermFreqArrayIterator(Iterator i) { this.i = i; @@ -62,4 +63,11 @@ public final class TermFreqArrayIterator implements TermFreqIterator { public Comparator getComparator() { return null; } + + @Override + public BytesRef surfaceForm() { + assert current != null; + surfaceSpare.copyBytes(current.surfaceForm); + return surfaceSpare; + } } \ No newline at end of file diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java index c883698..ee9968d 100644 --- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java @@ -97,6 +97,47 @@ public class AnalyzingSuggesterTest extends LuceneTestCase { assertEquals(6, results.get(2).value, 0.01F); } + public void testDifferentSurfaceForm() throws IOException { + TermFreq keys[] = new TermFreq[] { + new TermFreq("foo", 50, "Some Foo Some Bar"), + new TermFreq("bar", 10, "Some Bar Some Foo"), + new TermFreq("barbar", 12), + new TermFreq("barbara", 6) + }; + + AnalyzingSuggester suggester = new AnalyzingSuggester(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false)); + suggester.build(new TermFreqArrayIterator(keys)); + + // top N of 2, but only foo is available + List results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2); + assertEquals(1, results.size()); + assertEquals("Some Foo Some Bar", results.get(0).key.toString()); + assertEquals(50, results.get(0).value, 0.01F); + + results = suggester.lookup(_TestUtil.stringToCharSequence("bar", random()), false, 1); + assertEquals(1, results.size()); + assertEquals("barbar", results.get(0).key.toString()); + assertEquals(12, results.get(0).value, 0.01F); + + // top N Of 2 for 'b' + results = suggester.lookup(_TestUtil.stringToCharSequence("b", random()), false, 2); + assertEquals(2, results.size()); + assertEquals("barbar", results.get(0).key.toString()); + assertEquals(12, results.get(0).value, 0.01F); + assertEquals("Some Bar Some Foo", results.get(1).key.toString()); + assertEquals(10, results.get(1).value, 0.01F); + + // top N of 3 for 'ba' + results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random()), false, 3); + assertEquals(3, results.size()); + assertEquals("barbar", results.get(0).key.toString()); + assertEquals(12, results.get(0).value, 0.01F); + assertEquals("Some Bar Some Foo", results.get(1).key.toString()); + assertEquals(10, results.get(1).value, 0.01F); + assertEquals("barbara", results.get(2).key.toString()); + assertEquals(6, results.get(2).value, 0.01F); + } + // TODO: more tests /** * basic "standardanalyzer" test with stopword removal diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/CustomAnalyzingSuggesterTest.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/CustomAnalyzingSuggesterTest.java new file mode 100644 index 0000000..dd54f6a --- /dev/null +++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/CustomAnalyzingSuggesterTest.java @@ -0,0 +1,150 @@ +package org.apache.lucene.search.suggest.analyzing; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.List; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.search.suggest.TermFreq; +import org.apache.lucene.search.suggest.TermFreqArrayIterator; +import org.apache.lucene.search.suggest.Lookup.LookupResult; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.fst.PairOutputs.Pair; +import org.apache.lucene.util.fst.Util.MinResult; + + +/** + * A testcase that test the extensibility of {@link AnalyzingSuggester} to encode + * arbitrary surface forms + */ +public class CustomAnalyzingSuggesterTest extends LuceneTestCase { + + + public void testCustomSurfaceForm() throws IOException { + TermFreq keys[] = new TermFreq[] { + new TermFreq(new BytesRef("foo"), 50, encodeUserId("Foo", 1)), + new TermFreq(new BytesRef("bar"), 10, encodeUserId("Bar", 2)), + new TermFreq(new BytesRef("barbar"), 12, encodeUserId("Lucy Barbar", 3)), + new TermFreq(new BytesRef("barbara"), 6, encodeUserId("Barbara Streisand", 4)) + }; + + CustomAnalyzingSuggestor suggester = new CustomAnalyzingSuggestor(new MockAnalyzer(random(), MockTokenizer.KEYWORD, true)); + suggester.build(new TermFreqArrayIterator(keys)); + + // top N of 2, but only foo is available + List results = suggester.lookup(_TestUtil.stringToCharSequence("f", random()), false, 2); + assertEquals(1, results.size()); + assertEquals("Foo", results.get(0).key.toString()); + assertEquals(50, results.get(0).value, 0.01F); + assertUserId(results.get(0), 1); + + results = suggester.lookup(_TestUtil.stringToCharSequence("Bar", random()), false, 1); + assertEquals(1, results.size()); + assertEquals("Bar", results.get(0).key.toString()); + assertEquals(10, results.get(0).value, 0.01F); + assertUserId(results.get(0), 2); + + // top N Of 2 for 'b' + results = suggester.lookup(_TestUtil.stringToCharSequence("b", random()), false, 2); + assertEquals(2, results.size()); + assertEquals("Lucy Barbar", results.get(0).key.toString()); + assertEquals(12, results.get(0).value, 0.01F); + assertUserId(results.get(0), 3); + assertEquals("Bar", results.get(1).key.toString()); + assertEquals(10, results.get(1).value, 0.01F); + assertUserId(results.get(1), 2); + + // top N of 3 for 'ba' + results = suggester.lookup(_TestUtil.stringToCharSequence("ba", random()), false, 3); + assertEquals(3, results.size()); + assertEquals("Lucy Barbar", results.get(0).key.toString()); + assertEquals(12, results.get(0).value, 0.01F); + assertUserId(results.get(0), 3); + assertEquals("Bar", results.get(1).key.toString()); + assertEquals(10, results.get(1).value, 0.01F); + assertUserId(results.get(1), 2); + assertEquals("Barbara Streisand", results.get(2).key.toString()); + assertEquals(6, results.get(2).value, 0.01F); + assertUserId(results.get(2), 4); + + } + + private static void assertUserId(LookupResult res, int expectedId) { + assertTrue(res instanceof CustomLookupResult); + assertEquals(expectedId, ((CustomLookupResult)res).userId); + } + + private static BytesRef encodeUserId(String surfaceForm, int userId) { + BytesRef ref = new BytesRef(surfaceForm); + ref.grow(ref.length+4); + copyInt(ref, userId, ref.length); + ref.length +=4; + return ref; + } + + private static int parseInt(BytesRef b, int pos) { + return ((b.bytes[pos++] & 0xFF) << 24) | ((b.bytes[pos++] & 0xFF) << 16) + | ((b.bytes[pos++] & 0xFF) << 8) | (b.bytes[pos] & 0xFF); + } + + private static void copyInt(BytesRef ref, int value, int startOffset) { + ref.bytes[startOffset] = (byte) (value >> 24); + ref.bytes[startOffset + 1] = (byte) (value >> 16); + ref.bytes[startOffset + 2] = (byte) (value >> 8); + ref.bytes[startOffset + 3] = (byte) (value); + } + + + public static class CustomLookupResult extends LookupResult { + + int userId; + + public CustomLookupResult(CharSequence key, long value, int userId) { + super(key, value); + this.userId = userId; + } + } + + public static class CustomAnalyzingSuggestor extends AnalyzingSuggester { + + public CustomAnalyzingSuggestor(Analyzer analyzer) { + super(analyzer); + } + + @Override + protected LookupResult toLookupResult( + MinResult> completion, CharsRef spare) { + spare.grow(completion.output.output2.length); + UnicodeUtil.UTF8toUTF16(completion.output.output2.bytes, completion.output.output2.offset, completion.output.output2.length-4, spare); + return new CustomLookupResult(spare.toString(), decodeWeight(completion.output.output1), parseInt(completion.output.output2, completion.output.output2.length-4)); + } + + @Override + protected boolean isExactMatch(BytesRef key, BytesRef surfaceForm) { + BytesRef withoutId = new BytesRef(surfaceForm.bytes, surfaceForm.offset, surfaceForm.length-4); + return key.bytesEquals(withoutId); + } + + } +}