diff --git lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java index 187e327..5882fdf 100644 --- lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java +++ lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java @@ -59,7 +59,7 @@ public class HighFrequencyDictionary implements Dictionary { return new HighFrequencyIterator(); } - final class HighFrequencyIterator implements TermFreqIterator { + final class HighFrequencyIterator implements TermFreqPayloadIterator { private final BytesRef spare = new BytesRef(); private final TermsEnum termsEnum; private int minNumDocs; @@ -98,5 +98,15 @@ public class HighFrequencyDictionary implements Dictionary { } return null; } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } } } diff --git lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java index 5d3a59b..1a07d76 100644 --- lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java +++ lucene/suggest/src/java/org/apache/lucene/search/spell/TermFreqPayloadIterator.java @@ -17,20 +17,66 @@ package org.apache.lucene.search.spell; * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; // javadocs import org.apache.lucene.search.suggest.analyzing.FuzzySuggester; // javadocs import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefIterator; /** * Interface for enumerating term,weight,payload triples; * currently only {@link AnalyzingSuggester} and {@link * FuzzySuggester} support payloads. */ -public interface TermFreqPayloadIterator extends TermFreqIterator { +public interface TermFreqPayloadIterator extends BytesRefIterator { /** An arbitrary byte[] to record per suggestion. See * {@link LookupResult#payload} to retrieve the payload * for each suggestion. */ public BytesRef payload(); + + /** A term's weight, higher numbers mean better suggestions. */ + public long weight(); + + /** Returns true if the iterator has payloads */ + public boolean hasPayloads(); + + /** + * Wraps a BytesRefIterator as a TermFreqPayloadIterator, with all weights + * set to 1 and carries no payload + */ + public static class TermFreqPayloadIteratorWrapper implements TermFreqPayloadIterator { + private final BytesRefIterator wrapped; + + /** + * Creates a new wrapper, wrapping the specified iterator and + * specifying a weight value of 1 for all terms + * and nullifies associated payloads. + */ + public TermFreqPayloadIteratorWrapper(BytesRefIterator wrapped) { + this.wrapped = wrapped; + } + + @Override + public long weight() { + return 1; + } + + @Override + public BytesRef next() throws IOException { + return wrapped.next(); + } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } + } } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java index 6228667..c1c16ae 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/BufferingTermFreqIteratorWrapper.java @@ -18,7 +18,8 @@ package org.apache.lucene.search.suggest; */ import java.io.IOException; -import org.apache.lucene.search.spell.TermFreqIterator; + +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Counter; @@ -27,22 +28,30 @@ import org.apache.lucene.util.Counter; * This wrapper buffers incoming elements. * @lucene.experimental */ -public class BufferingTermFreqIteratorWrapper implements TermFreqIterator { +public class BufferingTermFreqIteratorWrapper implements TermFreqPayloadIterator { // TODO keep this for now /** buffered term entries */ protected BytesRefArray entries = new BytesRefArray(Counter.newCounter()); + /** buffered payload entries */ + protected BytesRefArray payloads = new BytesRefArray(Counter.newCounter()); /** current buffer position */ protected int curPos = -1; /** buffered weights, parallel with {@link #entries} */ protected long[] freqs = new long[1]; private final BytesRef spare = new BytesRef(); + private final BytesRef payloadSpare = new BytesRef(); + private final boolean hasPayloads; /** Creates a new iterator, buffering entries from the specified iterator */ - public BufferingTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { + public BufferingTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException { BytesRef spare; int freqIndex = 0; + hasPayloads = source.hasPayloads(); while((spare = source.next()) != null) { entries.append(spare); + if (hasPayloads) { + payloads.append(source.payload()); + } if (freqIndex >= freqs.length) { freqs = ArrayUtil.grow(freqs, freqs.length+1); } @@ -64,4 +73,17 @@ public class BufferingTermFreqIteratorWrapper implements TermFreqIterator { } return null; } + + @Override + public BytesRef payload() { + if (hasPayloads && curPos < payloads.size()) { + return payloads.get(payloadSpare, curPos); + } + return null; + } + + @Override + public boolean hasPayloads() { + return hasPayloads; + } } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java index 3519961..3a430c7 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java @@ -169,6 +169,11 @@ public class DocumentDictionary implements Dictionary { public BytesRef payload() { return currentPayload; } + + @Override + public boolean hasPayloads() { + return withPayload; + } } } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java index fa242ef..b03033b 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java @@ -21,7 +21,7 @@ package org.apache.lucene.search.suggest; import java.io.*; import org.apache.lucene.search.spell.Dictionary; -import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -57,11 +57,11 @@ public class FileDictionary implements Dictionary { } @Override - public TermFreqIterator getWordsIterator() { + public TermFreqPayloadIterator getWordsIterator() { return new FileIterator(); } - final class FileIterator implements TermFreqIterator { + final class FileIterator implements TermFreqPayloadIterator { private long curFreq; private final BytesRef spare = new BytesRef(); @@ -98,5 +98,15 @@ public class FileDictionary implements Dictionary { return null; } } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } } } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java index a1c64d3..2cd5377 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java @@ -24,7 +24,7 @@ import java.util.Comparator; import java.util.List; import org.apache.lucene.search.spell.Dictionary; -import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; import org.apache.lucene.util.PriorityQueue; @@ -159,20 +159,20 @@ public abstract class Lookup { */ public void build(Dictionary dict) throws IOException { BytesRefIterator it = dict.getWordsIterator(); - TermFreqIterator tfit; - if (it instanceof TermFreqIterator) { - tfit = (TermFreqIterator)it; + TermFreqPayloadIterator tfit; + if (it instanceof TermFreqPayloadIterator) { + tfit = (TermFreqPayloadIterator)it; } else { - tfit = new TermFreqIterator.TermFreqIteratorWrapper(it); + tfit = new TermFreqPayloadIterator.TermFreqPayloadIteratorWrapper(it); } build(tfit); } /** - * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqIterator}. + * Builds up a new internal {@link Lookup} representation based on the given {@link TermFreqPayloadIterator}. * The implementation might re-sort the data internally. */ - public abstract void build(TermFreqIterator tfit) throws IOException; + public abstract void build(TermFreqPayloadIterator tfit) throws IOException; /** * Look up a key and return possible completion for this key. diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java index 53c4212..bf6207f 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java @@ -21,7 +21,7 @@ import java.io.File; import java.io.IOException; import java.util.Comparator; -import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.search.suggest.Sort.ByteSequencesReader; import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter; import org.apache.lucene.store.ByteArrayDataInput; @@ -34,9 +34,9 @@ import org.apache.lucene.util.IOUtils; * This wrapper buffers incoming elements and makes sure they are sorted based on given comparator. * @lucene.experimental */ -public class SortedTermFreqIteratorWrapper implements TermFreqIterator { +public class SortedTermFreqIteratorWrapper implements TermFreqPayloadIterator { - private final TermFreqIterator source; + private final TermFreqPayloadIterator source; private File tempInput; private File tempSorted; private final ByteSequencesReader reader; @@ -50,7 +50,7 @@ public class SortedTermFreqIteratorWrapper implements TermFreqIterator { * Creates a new sorted wrapper, using {@link * BytesRef#getUTF8SortedAsUnicodeComparator} for * sorting. */ - public SortedTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { + public SortedTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException { this(source, BytesRef.getUTF8SortedAsUnicodeComparator()); } @@ -58,7 +58,10 @@ public class SortedTermFreqIteratorWrapper implements TermFreqIterator { * Creates a new sorted wrapper, sorting by BytesRef * (ascending) then cost (ascending). */ - public SortedTermFreqIteratorWrapper(TermFreqIterator source, Comparator comparator) throws IOException { + public SortedTermFreqIteratorWrapper(TermFreqPayloadIterator source, Comparator comparator) throws IOException { + if (source.hasPayloads()) { + throw new IllegalArgumentException("SortedTermFreqIteratorWrapper does not support payloads"); + } this.source = source; this.comparator = comparator; this.reader = sort(); @@ -93,6 +96,18 @@ public class SortedTermFreqIteratorWrapper implements TermFreqIterator { return weight; } + @Override + public BytesRef payload() { + //TODO: support encoding/decoding payloads along with terms+weight + return null; + } + + @Override + public boolean hasPayloads() { + // TODO: after supporting payloads use source.hasPayloads() instead + return false; + } + /** Sortes by BytesRef (ascending) then cost (ascending). */ private final Comparator tieBreakByCostComparator = new Comparator() { diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java index c242195..8b3d2e0 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/UnsortedTermFreqIteratorWrapper.java @@ -20,7 +20,7 @@ package org.apache.lucene.search.suggest; import java.io.IOException; import java.util.Random; -import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.BytesRef; /** @@ -33,11 +33,12 @@ public class UnsortedTermFreqIteratorWrapper extends BufferingTermFreqIteratorWr private final int[] ords; private int currentOrd = -1; private final BytesRef spare = new BytesRef(); + private final BytesRef payloadSpare = new BytesRef(); /** * Creates a new iterator, wrapping the specified iterator and * returning elements in a random order. */ - public UnsortedTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { + public UnsortedTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException { super(source); ords = new int[entries.size()]; Random random = new Random(); @@ -64,4 +65,13 @@ public class UnsortedTermFreqIteratorWrapper extends BufferingTermFreqIteratorWr } return null; } + + @Override + public BytesRef payload() { + if (hasPayloads() && curPos < payloads.size()) { + return payloads.get(payloadSpare, (currentOrd = ords[curPos])); + } + return null; + + } } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java index 632023d..27d73b8 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java @@ -65,7 +65,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.spell.TermFreqIterator; import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs import org.apache.lucene.search.suggest.Lookup; @@ -176,19 +175,14 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable { } @Override - public void build(TermFreqIterator iter) throws IOException { + public void build(TermFreqPayloadIterator iter) throws IOException { if (searcher != null) { searcher.getIndexReader().close(); searcher = null; } - TermFreqPayloadIterator payloads; - if (iter instanceof TermFreqPayloadIterator) { - payloads = (TermFreqPayloadIterator) iter; - } else { - payloads = null; - } + Directory dirTmp = getDirectory(new File(indexPath.toString() + ".tmp")); IndexWriter w = null; @@ -236,7 +230,7 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable { doc.add(weightField); Field payloadField; - if (payloads != null) { + if (iter.hasPayloads()) { payloadField = new BinaryDocValuesField("payloads", new BytesRef()); doc.add(payloadField); } else { @@ -250,8 +244,8 @@ public class AnalyzingInfixSuggester extends Lookup implements Closeable { textGramField.setStringValue(textString); textDVField.setBytesValue(text); weightField.setLongValue(iter.weight()); - if (payloads != null) { - payloadField.setBytesValue(payloads.payload()); + if (iter.hasPayloads()) { + payloadField.setBytesValue(iter.payload()); } w.addDocument(doc); } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java index 77f0f1c..62d2ee1 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java @@ -381,19 +381,13 @@ public class AnalyzingSuggester extends Lookup { } @Override - public void build(TermFreqIterator iterator) throws IOException { + public void build(TermFreqPayloadIterator iterator) throws IOException { String prefix = getClass().getSimpleName(); File directory = Sort.defaultTempDir(); File tempInput = File.createTempFile(prefix, ".input", directory); File tempSorted = File.createTempFile(prefix, ".sorted", directory); - TermFreqPayloadIterator payloads; - if (iterator instanceof TermFreqPayloadIterator) { - payloads = (TermFreqPayloadIterator) iterator; - } else { - payloads = null; - } - hasPayloads = payloads != null; + hasPayloads = iterator.hasPayloads(); Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput); Sort.ByteSequencesReader reader = null; @@ -432,7 +426,7 @@ public class AnalyzingSuggester extends Lookup { if (surfaceForm.length > (Short.MAX_VALUE-2)) { throw new IllegalArgumentException("cannot handle surface form > " + (Short.MAX_VALUE-2) + " in length (got " + surfaceForm.length + ")"); } - payload = payloads.payload(); + payload = iterator.payload(); // payload + surfaceLength (short) requiredLength += payload.length + 2; } else { @@ -470,7 +464,7 @@ public class AnalyzingSuggester extends Lookup { writer.close(); // Sort all input/output pairs (required by FST.Builder): - new Sort(new AnalyzingComparator(payloads != null)).sort(tempInput, tempSorted); + new Sort(new AnalyzingComparator(hasPayloads)).sort(tempInput, tempSorted); // Free disk space: tempInput.delete(); diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java index d2f652d..cee929b 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java @@ -54,7 +54,6 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.spell.TermFreqIterator; import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.Sort; @@ -274,15 +273,15 @@ public class FreeTextSuggester extends Lookup { } @Override - public void build(TermFreqIterator iterator) throws IOException { + public void build(TermFreqPayloadIterator iterator) throws IOException { build(iterator, IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB); } /** Build the suggest index, using up to the specified * amount of temporary RAM while building. Note that * the weights for the suggestions are ignored. */ - public void build(TermFreqIterator iterator, double ramBufferSizeMB) throws IOException { - if (iterator instanceof TermFreqPayloadIterator) { + public void build(TermFreqPayloadIterator iterator, double ramBufferSizeMB) throws IOException { + if (iterator.hasPayloads()) { throw new IllegalArgumentException("payloads are not supported"); } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index 2f4fe05..a180138 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -43,7 +43,7 @@ import org.apache.lucene.util.fst.NoOutputs; * An adapter from {@link Lookup} API to {@link FSTCompletion}. * *

This adapter differs from {@link FSTCompletion} in that it attempts - * to discretize any "weights" as passed from in {@link TermFreqIterator#weight()} + * to discretize any "weights" as passed from in {@link TermFreqPayloadIterator#weight()} * to match the number of buckets. For the rationale for bucketing, see * {@link FSTCompletion}. * @@ -96,7 +96,7 @@ public class FSTCompletionLookup extends Lookup { /** * This constructor prepares for creating a suggested FST using the - * {@link #build(TermFreqIterator)} method. The number of weight + * {@link #build(TermFreqPayloadIterator)} method. The number of weight * discretization buckets is set to {@link FSTCompletion#DEFAULT_BUCKETS} and * exact matches are promoted to the top of the suggestions list. */ @@ -106,7 +106,7 @@ public class FSTCompletionLookup extends Lookup { /** * This constructor prepares for creating a suggested FST using the - * {@link #build(TermFreqIterator)} method. + * {@link #build(TermFreqPayloadIterator)} method. * * @param buckets * The number of weight discretization buckets (see @@ -141,8 +141,8 @@ public class FSTCompletionLookup extends Lookup { } @Override - public void build(TermFreqIterator tfit) throws IOException { - if (tfit instanceof TermFreqPayloadIterator) { + public void build(TermFreqPayloadIterator tfit) throws IOException { + if (tfit.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } File tempInput = File.createTempFile( diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java index f634bee..6c252dc 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java @@ -25,7 +25,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; -import org.apache.lucene.search.spell.TermFreqIterator; import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.Sort.ByteSequencesWriter; @@ -93,12 +92,12 @@ public class WFSTCompletionLookup extends Lookup { } @Override - public void build(TermFreqIterator iterator) throws IOException { - if (iterator instanceof TermFreqPayloadIterator) { + public void build(TermFreqPayloadIterator iterator) throws IOException { + if (iterator.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } BytesRef scratch = new BytesRef(); - TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator); + TermFreqPayloadIterator iter = new WFSTTermFreqIteratorWrapper(iterator); IntsRef scratchInts = new IntsRef(); BytesRef previous = null; PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(); @@ -257,7 +256,7 @@ public class WFSTCompletionLookup extends Lookup { private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqIteratorWrapper { - WFSTTermFreqIteratorWrapper(TermFreqIterator source) throws IOException { + WFSTTermFreqIteratorWrapper(TermFreqPayloadIterator source) throws IOException { super(source); } diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java index 558e115..6ec5f9b 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java @@ -47,13 +47,13 @@ public class JaspellLookup extends Lookup { /** * Creates a new empty trie - * @see #build(TermFreqIterator) + * @see #build(TermFreqPayloadIterator) * */ public JaspellLookup() {} @Override - public void build(TermFreqIterator tfit) throws IOException { - if (tfit instanceof TermFreqPayloadIterator) { + public void build(TermFreqPayloadIterator tfit) throws IOException { + if (tfit.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } trie = new JaspellTernarySearchTrie(); diff --git lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java index 852ebb5..b9cc6fb 100644 --- lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java @@ -46,13 +46,13 @@ public class TSTLookup extends Lookup { /** * Creates a new TSTLookup with an empty Ternary Search Tree. - * @see #build(TermFreqIterator) + * @see #build(TermFreqPayloadIterator) */ public TSTLookup() {} @Override - public void build(TermFreqIterator tfit) throws IOException { - if (tfit instanceof TermFreqPayloadIterator) { + public void build(TermFreqPayloadIterator tfit) throws IOException { + if (tfit.hasPayloads()) { throw new IllegalArgumentException("this suggester doesn't support payloads"); } root = new TernaryTreeNode(); diff --git lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java index d77fa5c..2845da1 100644 --- lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java +++ lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqArrayIterator.java @@ -21,12 +21,13 @@ import java.util.Arrays; import java.util.Iterator; import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.BytesRef; /** - * A {@link TermFreqIterator} over a sequence of {@link TermFreq}s. + * A {@link TermFreqPayloadIterator} over a sequence of {@link TermFreq}s. */ -public final class TermFreqArrayIterator implements TermFreqIterator { +public final class TermFreqArrayIterator implements TermFreqPayloadIterator { private final Iterator i; private TermFreq current; private final BytesRef spare = new BytesRef(); @@ -57,4 +58,14 @@ public final class TermFreqArrayIterator implements TermFreqIterator { } return null; } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } } \ No newline at end of file diff --git lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java index 5bfb073..a55a140 100644 --- lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java +++ lucene/suggest/src/test/org/apache/lucene/search/suggest/TermFreqPayloadArrayIterator.java @@ -25,7 +25,7 @@ import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.util.BytesRef; /** - * A {@link TermFreqIterator} over a sequence of {@link TermFreq}s. + * A {@link TermFreqPayloadIterator} over a sequence of {@link TermFreq}s. */ public final class TermFreqPayloadArrayIterator implements TermFreqPayloadIterator { private final Iterator i; @@ -63,4 +63,9 @@ public final class TermFreqPayloadArrayIterator implements TermFreqPayloadIterat public BytesRef payload() { return current.payload; } + + @Override + public boolean hasPayloads() { + return true; + } } \ No newline at end of file diff --git lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java index 3209b1a..bfea92c 100644 --- lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java +++ lucene/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java @@ -24,6 +24,7 @@ import java.util.Random; import java.util.TreeMap; import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -34,7 +35,7 @@ import org.apache.lucene.util._TestUtil; public class TestTermFreqIterator extends LuceneTestCase { public void testEmpty() throws Exception { TermFreqArrayIterator iterator = new TermFreqArrayIterator(new TermFreq[0]); - TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(iterator, BytesRef.getUTF8SortedAsUnicodeComparator()); + TermFreqPayloadIterator wrapper = new SortedTermFreqIteratorWrapper(iterator, BytesRef.getUTF8SortedAsUnicodeComparator()); assertNull(wrapper.next()); wrapper = new UnsortedTermFreqIteratorWrapper(iterator); assertNull(wrapper.next()); @@ -59,7 +60,7 @@ public class TestTermFreqIterator extends LuceneTestCase { } // test the sorted iterator wrapper - TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), comparator); + TermFreqPayloadIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), comparator); Iterator> expected = sorted.entrySet().iterator(); while (expected.hasNext()) { Map.Entry entry = expected.next(); diff --git lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java index 7ed9ccb..ff3497b 100644 --- lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java +++ lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java @@ -41,7 +41,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.StopFilter; import org.apache.lucene.analysis.util.CharArraySet; import org.apache.lucene.document.Document; -import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.spell.TermFreqPayloadIterator; import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.search.suggest.TermFreq; import org.apache.lucene.search.suggest.TermFreqArrayIterator; @@ -136,7 +136,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { // Skip header: lfd.nextDoc(); FreeTextSuggester sug = new FreeTextSuggester(new MockAnalyzer(random())); - sug.build(new TermFreqIterator() { + sug.build(new TermFreqPayloadIterator() { private int count; @@ -161,6 +161,16 @@ public class TestFreeTextSuggester extends LuceneTestCase { } return new BytesRef(doc.get("body")); } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } }); if (VERBOSE) { System.out.println(sug.sizeInBytes() + " bytes"); @@ -320,7 +330,7 @@ public class TestFreeTextSuggester extends LuceneTestCase { // Build suggester model: FreeTextSuggester sug = new FreeTextSuggester(a, a, grams, (byte) 0x20); - sug.build(new TermFreqIterator() { + sug.build(new TermFreqPayloadIterator() { int upto; @Override @@ -342,6 +352,16 @@ public class TestFreeTextSuggester extends LuceneTestCase { public long weight() { return random().nextLong(); } + + @Override + public BytesRef payload() { + return null; + } + + @Override + public boolean hasPayloads() { + return false; + } }); // Build inefficient but hopefully correct model: