As the Lucene similarity that is used to fetch the most relevant n-grammed terms @@ -224,12 +238,38 @@ * @throws IOException if the underlying index throws an {@link IOException} * @throws AlreadyClosedException if the Spellchecker is already closed * @return String[] + * + * @see #suggestSimilar(String, int, org.apache.lucene.index.IndexReader, String, boolean, float) */ public String[] suggestSimilar(String word, int numSug) throws IOException { return this.suggestSimilar(word, numSug, null, null, false); } /** + * Suggest similar words. + * + *
As the Lucene similarity that is used to fetch the most relevant n-grammed terms + * is not the same as the edit distance strategy used to calculate the best + * matching spell-checked word from the hits that Lucene found, one usually has + * to retrieve a couple of numSug's in order to get the true best match. + * + *
I.e. if numSug == 1, don't count on that suggestion being the best one. + * Thus, you should set this value to at least 5 for a good suggestion. + * + * @param word the word you want a spell check done on + * @param numSug the number of suggested words + * @param accuracy The minimum score a suggestion must have in order to qualify for inclusion in the results + * @throws IOException if the underlying index throws an {@link IOException} + * @throws AlreadyClosedException if the Spellchecker is already closed + * @return String[] + * + * @see #suggestSimilar(String, int, org.apache.lucene.index.IndexReader, String, boolean, float) + */ + public String[] suggestSimilar(String word, int numSug, float accuracy) throws IOException { + return this.suggestSimilar(word, numSug, null, null, false, accuracy); + } + + /** * Suggest similar words (optionally restricted to a field of an index). * *
As the Lucene similarity that is used to fetch the most relevant n-grammed terms @@ -240,6 +280,8 @@ *
I.e. if numSug == 1, don't count on that suggestion being the best one. * Thus, you should set this value to at least 5 for a good suggestion. * + *
Uses the {@link #getAccuracy()} value passed into the constructor as the accuracy. + * * @param word the word you want a spell check done on * @param numSug the number of suggested words * @param ir the indexReader of the user index (can be null see field param) @@ -252,74 +294,107 @@ * @return String[] the sorted list of the suggest words with these 2 criteria: * first criteria: the edit distance, second criteria (only if restricted mode): the popularity * of the suggest words in the field of the user index + * + * @see #suggestSimilar(String, int, org.apache.lucene.index.IndexReader, String, boolean, float) */ public String[] suggestSimilar(String word, int numSug, IndexReader ir, String field, boolean morePopular) throws IOException { + return suggestSimilar(word, numSug, ir, field, morePopular, accuracy); + } + + + /** + * Suggest similar words (optionally restricted to a field of an index). + * + *
As the Lucene similarity that is used to fetch the most relevant n-grammed terms + * is not the same as the edit distance strategy used to calculate the best + * matching spell-checked word from the hits that Lucene found, one usually has + * to retrieve a couple of numSug's in order to get the true best match. + * + *
I.e. if numSug == 1, don't count on that suggestion being the best one. + * Thus, you should set this value to at least 5 for a good suggestion. + * + * @param word the word you want a spell check done on + * @param numSug the number of suggested words + * @param ir the indexReader of the user index (can be null see field param) + * @param field the field of the user index: if field is not null, the suggested + * words are restricted to the words present in this field. + * @param morePopular return only the suggest words that are as frequent or more frequent than the searched word + * (only if restricted mode = (indexReader!=null and field!=null) + * @param accuracy The minimum score a suggestion must have in order to qualify for inclusion in the results + * @throws IOException if the underlying index throws an {@link IOException} + * @throws AlreadyClosedException if the Spellchecker is already closed + * @return String[] the sorted list of the suggest words with these 2 criteria: + * first criteria: the edit distance, second criteria (only if restricted mode): the popularity + * of the suggest words in the field of the user index + */ + public String[] suggestSimilar(String word, int numSug, IndexReader ir, + String field, boolean morePopular, float accuracy) throws IOException { // obtainSearcher calls ensureOpen final IndexSearcher indexSearcher = obtainSearcher(); try{ - float min = this.minScore; + final int lengthWord = word.length(); - + final int freq = (ir != null && field != null) ? ir.docFreq(new Term(field, word)) : 0; final int goalFreq = (morePopular && ir != null && field != null) ? freq : 0; // if the word exists in the real index and we don't care for word frequency, return the word itself if (!morePopular && freq > 0) { return new String[] { word }; } - + BooleanQuery query = new BooleanQuery(); String[] grams; String key; - + for (int ng = getMin(lengthWord); ng <= getMax(lengthWord); ng++) { - + key = "gram" + ng; // form key - + grams = formGrams(word, ng); // form word into ngrams (allow dups too) - + if (grams.length == 0) { continue; // hmm } - + if (bStart > 0) { // should we boost prefixes? add(query, "start" + ng, grams[0], bStart); // matches start of word - + } if (bEnd > 0) { // should we boost suffixes add(query, "end" + ng, grams[grams.length - 1], bEnd); // matches end of word - + } for (int i = 0; i < grams.length; i++) { add(query, key, grams[i]); } } - + int maxHits = 10 * numSug; - + // System.out.println("Q: " + query); ScoreDoc[] hits = indexSearcher.search(query, null, maxHits).scoreDocs; // System.out.println("HITS: " + hits.length()); SuggestWordQueue sugQueue = new SuggestWordQueue(numSug, comparator); - + // go thru more than 'maxr' matches in case the distance filter triggers int stop = Math.min(hits.length, maxHits); SuggestWord sugWord = new SuggestWord(); for (int i = 0; i < stop; i++) { - + sugWord.string = indexSearcher.doc(hits[i].doc).get(F_WORD); // get orig word - + // don't suggest a word for itself, that would be silly if (sugWord.string.equals(word)) { continue; } - + // edit distance sugWord.score = sd.getDistance(word,sugWord.string); - if (sugWord.score < min) { + if (sugWord.score < accuracy) { continue; } - + if (ir != null && field != null) { // use the user index sugWord.freq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index // don't suggest a word that is not present in the field @@ -330,23 +405,22 @@ sugQueue.insertWithOverflow(sugWord); if (sugQueue.size() == numSug) { // if queue full, maintain the minScore score - min = sugQueue.top().score; + accuracy = sugQueue.top().score; } sugWord = new SuggestWord(); } - + // convert to array string String[] list = new String[sugQueue.size()]; for (int i = sugQueue.size() - 1; i >= 0; i--) { list[i] = sugQueue.pop().string; } - + return list; } finally { releaseSearcher(indexSearcher); } } - /** * Add a clause to a boolean query. */