Index: solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (working copy) @@ -203,7 +203,7 @@ } if (!foundOriginal) { SuggestWord orig = new SuggestWord(); - orig.freq = freq; + orig.docFreq = freq; orig.string = tokenText; suggestionsWithOrig[0] = orig; suggestions = suggestionsWithOrig; @@ -214,7 +214,7 @@ result.add(token, empty); } else { for (SuggestWord suggestion : suggestions) { - result.add(token, suggestion.string, suggestion.freq); + result.add(token, suggestion.string, suggestion.docFreq); } } } Index: solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java (working copy) @@ -239,9 +239,9 @@ firstOne = false; sb.append(word.string); if (sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) { - freq = Math.max(freq, word.freq); + freq = Math.max(freq, word.docFreq); } else { - freq += word.freq; + freq += word.docFreq; } } breakSuggestionList.add(new ResultEntry(tokenArr[i], sb.toString(), @@ -268,7 +268,7 @@ Token token = new Token(sb.toString(), tokenArr[firstTermIndex] .startOffset(), tokenArr[lastTermIndex].endOffset()); combineSuggestionList.add(new ResultEntry(token, cs.suggestion.string, - cs.suggestion.freq)); + cs.suggestion.docFreq)); } } Index: solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java (working copy) @@ -134,7 +134,7 @@ Integer o = mergeData.origVsFreq.get(original); if (o != null) result.addFrequency(token, o); for (SuggestWord word : suggestions) - result.add(token, word.string, word.freq); + result.add(token, word.string, word.docFreq); } else { List words = new ArrayList(sugQueue.size()); for (SuggestWord word : suggestions) words.add(word.string); Index: solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (working copy) @@ -378,7 +378,7 @@ if (suggestion.getAlternativeFrequencies() != null && suggestion.getAlternativeFrequencies().size() > 0) { Integer freq = suggestion.getAlternativeFrequencies().get(i); - if (freq != null) sug.freq += freq; + if (freq != null) sug.docFreq += freq; } } } Index: lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java =================================================================== --- lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java (revision 1453360) +++ lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java (working copy) @@ -210,8 +210,8 @@ Assert.assertTrue(sw[0][1].string.equals("thousand")); Assert.assertTrue(sw[0][0].score == 1); Assert.assertTrue(sw[0][1].score == 1); - Assert.assertTrue(sw[0][1].freq>1); - Assert.assertTrue(sw[0][0].freq>sw[0][1].freq); + Assert.assertTrue(sw[0][1].docFreq>1); + Assert.assertTrue(sw[0][0].docFreq>sw[0][1].docFreq); Assert.assertTrue(sw[1].length==3); Assert.assertTrue(sw[1][0].string.equals("one")); Assert.assertTrue(sw[1][1].string.equals("thou")); @@ -219,9 +219,9 @@ Assert.assertTrue(sw[1][0].score == 2); Assert.assertTrue(sw[1][1].score == 2); Assert.assertTrue(sw[1][2].score == 2); - Assert.assertTrue(sw[1][0].freq>1); - Assert.assertTrue(sw[1][1].freq==1); - Assert.assertTrue(sw[1][2].freq==1); + Assert.assertTrue(sw[1][0].docFreq>1); + Assert.assertTrue(sw[1][1].docFreq==1); + Assert.assertTrue(sw[1][2].docFreq==1); } { Term term = new Term("numbers", "onethousandonehundredeleven"); Index: lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.FuzzyTermsEnum; import org.apache.lucene.search.MaxNonCompetitiveBoostAttribute; @@ -36,7 +37,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashSet; -import java.util.Locale; import java.util.PriorityQueue; /** @@ -74,13 +74,13 @@ private float accuracy = SpellChecker.DEFAULT_ACCURACY; /** value in [0..1] (or absolute number >=1) representing the minimum * number of documents (of the total) where a term should appear. */ - private float thresholdFrequency = 0f; + private double thresholdFrequency = 0; /** minimum length of a query word to return suggestions */ private int minQueryLength = 4; /** value in [0..1] (or absolute number >=1) representing the maximum * number of documents (of the total) a query term can appear in to * be corrected. */ - private float maxQueryFrequency = 0.01f; + private double maxQueryFrequency = 0.01f; /** true if the spellchecker should lowercase terms */ private boolean lowerCaseTerms = true; /** the comparator to use */ @@ -163,7 +163,7 @@ /** * Get the minimal threshold of documents a term must appear for a match */ - public float getThresholdFrequency() { + public double getThresholdFrequency() { return thresholdFrequency; } @@ -204,7 +204,7 @@ * Get the maximum threshold of documents a query term can appear in order * to provide suggestions. */ - public float getMaxQueryFrequency() { + public double getMaxQueryFrequency() { return maxQueryFrequency; } @@ -317,54 +317,88 @@ */ public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir, SuggestMode suggestMode, float accuracy) throws IOException { - final CharsRef spare = new CharsRef(); - String text = term.text(); - if (minQueryLength > 0 && text.codePointCount(0, text.length()) < minQueryLength) + final String field = term.field(); + final Terms terms = MultiFields.getTerms(ir, field); + if (terms == null) { + return new SuggestWord[0]; + } + final FieldStatistics stats = new DocFreqFieldStatistics(terms, field, ir); + final CharsRef spare = new CharsRef(); + return suggestSimilar(term.bytes(), numSug, stats, suggestMode, accuracy, spare); + } + + /** + * Suggest similar words. + * + *

Unlike {@link SpellChecker}, the similarity used to fetch the most + * relevant terms is an edit distance, therefore typically a low value + * for numSug will work very well. + * + * @param utf8Text the term you want to spell check on + * @param numSug the maximum number of suggested words + * @param stats the {@link FieldStatistics} to use to find the relevant terms + * @param suggestMode specifies when to return suggested words + * @param accuracy return only suggested words that match with this similarity + * @return sorted list of the suggested words according to the comparator + * @throws IOException If there is a low-level I/O error. + */ + public SuggestWord[] suggestSimilar(BytesRef utf8Text, int numSug, final FieldStatistics stats, + SuggestMode suggestMode, float accuracy, CharsRef spare) throws IOException { + UnicodeUtil.UTF8toUTF16(utf8Text, spare); + if (minQueryLength > 0 && Character.codePointCount(spare, 0, spare.length) < minQueryLength) { return new SuggestWord[0]; + } if (lowerCaseTerms) { - term = new Term(term.field(), text.toLowerCase(Locale.ROOT)); - } + for (int i = 0; i < spare.length;) { + i += Character.toChars( + Character.toLowerCase( + Character.codePointAt(spare.chars, i)), spare.chars, i); + } + utf8Text = new BytesRef(spare); + } - int docfreq = ir.docFreq(term); + final String utf16Term = spare.toString(); + long termFrequency = stats.freq(utf8Text); - if (suggestMode==SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && docfreq > 0) { + if (suggestMode == SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && termFrequency > 0) { return new SuggestWord[0]; } - int maxDoc = ir.maxDoc(); + long totalFrequency = stats.totalFreq(); - if (maxQueryFrequency >= 1f && docfreq > maxQueryFrequency) { + if (maxQueryFrequency >= 1f && termFrequency > maxQueryFrequency) { return new SuggestWord[0]; - } else if (docfreq > (int) Math.ceil(maxQueryFrequency * (float)maxDoc)) { + } else if (termFrequency > (long) Math.ceil(maxQueryFrequency * (double)totalFrequency)) { return new SuggestWord[0]; } - if (suggestMode!=SuggestMode.SUGGEST_MORE_POPULAR) docfreq = 0; + if (suggestMode != SuggestMode.SUGGEST_MORE_POPULAR) { + termFrequency = 0; + } if (thresholdFrequency >= 1f) { - docfreq = Math.max(docfreq, (int) thresholdFrequency); + termFrequency = Math.max(termFrequency, (long) thresholdFrequency); } else if (thresholdFrequency > 0f) { - docfreq = Math.max(docfreq, (int)(thresholdFrequency * (float)maxDoc)-1); + termFrequency = Math.max(termFrequency, (long)(thresholdFrequency * (double)totalFrequency)-1); } - Collection terms = null; - int inspections = numSug * maxInspections; + final int inspections = numSug * maxInspections; // try ed=1 first, in case we get lucky - terms = suggestSimilar(term, inspections, ir, docfreq, 1, accuracy, spare); - if (maxEdits > 1 && terms.size() < inspections) { - HashSet moreTerms = new HashSet(); - moreTerms.addAll(terms); - moreTerms.addAll(suggestSimilar(term, inspections, ir, docfreq, maxEdits, accuracy, spare)); - terms = moreTerms; + Collection scoreTerms = suggestSimilar(utf8Text, utf16Term, inspections, stats, termFrequency, 1, accuracy, spare); + if (maxEdits > 1 && scoreTerms.size() < inspections) { + final HashSet moreTerms = new HashSet(); + moreTerms.addAll(scoreTerms); + moreTerms.addAll(suggestSimilar(utf8Text, utf16Term, inspections, stats, termFrequency, maxEdits, accuracy, spare)); + scoreTerms = moreTerms; } // create the suggestword response, sort it, and trim it to size. - SuggestWord suggestions[] = new SuggestWord[terms.size()]; + final SuggestWord suggestions[] = new SuggestWord[scoreTerms.size()]; int index = suggestions.length - 1; - for (ScoreTerm s : terms) { + for (ScoreTerm s : scoreTerms) { SuggestWord suggestion = new SuggestWord(); if (s.termAsString == null) { UnicodeUtil.UTF8toUTF16(s.term, spare); @@ -372,7 +406,8 @@ } suggestion.string = s.termAsString; suggestion.score = s.score; - suggestion.freq = s.docfreq; + suggestion.totalTermFreq = s.totalTermFreq; + suggestion.docFreq = s.docfreq; suggestions[index--] = suggestion; } @@ -380,38 +415,37 @@ if (numSug < suggestions.length) { SuggestWord trimmed[] = new SuggestWord[numSug]; System.arraycopy(suggestions, 0, trimmed, 0, numSug); - suggestions = trimmed; + return trimmed; } return suggestions; } - /** * Provide spelling corrections based on several parameters. * - * @param term The term to suggest spelling corrections for + * @param utf8Term The UTF-8 term to suggest spelling corrections for + * @param utf16Term The UTF-16 term to suggest spelling corrections for * @param numSug The maximum number of spelling corrections - * @param ir The index reader to fetch the candidate spelling corrections from - * @param docfreq The minimum document frequency a potential suggestion need to have in order to be included + * @param stats The statistics for the field to fetch corrections from + * @param totalFrequency The minimum total term frequency a potential suggestion need to have in order to be included * @param editDistance The maximum edit distance candidates are allowed to have * @param accuracy The minimum accuracy a suggested spelling correction needs to have in order to be included * @param spare a chars scratch * @return a collection of spelling corrections sorted by ScoreTerm's natural order. * @throws IOException If I/O related errors occur */ - protected Collection suggestSimilar(Term term, int numSug, IndexReader ir, int docfreq, int editDistance, + protected Collection suggestSimilar(BytesRef utf8Term, String utf16Term, int numSug, FieldStatistics stats, long totalFrequency, int editDistance, float accuracy, final CharsRef spare) throws IOException { AttributeSource atts = new AttributeSource(); MaxNonCompetitiveBoostAttribute maxBoostAtt = atts.addAttribute(MaxNonCompetitiveBoostAttribute.class); - Terms terms = MultiFields.getTerms(ir, term.field()); + final Terms terms = stats.getTerms(); if (terms == null) { return Collections.emptyList(); } - FuzzyTermsEnum e = new FuzzyTermsEnum(terms, atts, term, editDistance, Math.max(minPrefix, editDistance-1), true); + FuzzyTermsEnum e = new FuzzyTermsEnum(stats.getTerms(), atts, utf16Term, editDistance, Math.max(minPrefix, editDistance-1), true); final PriorityQueue stQueue = new PriorityQueue(); - BytesRef queryTerm = new BytesRef(term.text()); BytesRef candidateTerm; ScoreTerm st = new ScoreTerm(); BoostAttribute boostAtt = @@ -423,15 +457,13 @@ continue; // ignore exact match of the same term - if (queryTerm.bytesEquals(candidateTerm)) + if (utf8Term.bytesEquals(candidateTerm)) continue; - int df = e.docFreq(); - - // check docFreq if required - if (df <= docfreq) + + // check ttf if required + if (stats.freq(e) <= totalFrequency) continue; - final float score; final String termAsString; if (distance == INTERNAL_LEVENSHTEIN) { @@ -442,7 +474,7 @@ } else { UnicodeUtil.UTF8toUTF16(candidateTerm, spare); termAsString = spare.toString(); - score = distance.getDistance(term.text(), termAsString); + score = distance.getDistance(utf16Term, termAsString); } if (score < accuracy) @@ -451,7 +483,8 @@ // add new entry in PQ st.term = BytesRef.deepCopyOf(candidateTerm); st.boost = boost; - st.docfreq = df; + st.totalTermFreq = e.totalTermFreq();; + st.docfreq = e.docFreq(); st.termAsString = termAsString; st.score = score; stQueue.offer(st); @@ -479,9 +512,14 @@ public float boost; /** - * The df of the spellcheck correction. + * The document frequency of the spellcheck correction. */ public int docfreq; + + /** + * The total term frequency of the spellcheck correction or -1 if this statistics are not available + */ + public long totalTermFreq; /** * The spellcheck correction represented as string, can be null. @@ -529,4 +567,139 @@ return true; } } + + /** + * A simple wrapper around at {@link Terms} instance providing syntactic sugar + * to access term frequency statistics per fiedl. + * @lucene.internal + */ + public abstract static class FieldStatistics { + + private final Terms terms; + private final long totalFreq; + private TermsEnum termsEnum; + private final String field; + + /** + * Creates a new {@link FieldStatistics} instance + * @param terms the {@link Terms} to wrap + * @param field the field + * @param totalFrequency the total frequency for this field + */ + public FieldStatistics(Terms terms, String field, long totalFrequency) { + this.terms = terms; + this.totalFreq = totalFrequency; + this.field = field; + } + + /** + * Returns the {@link Terms} instance for this field. + * @return the {@link Terms} instance for this field. + */ + public Terms getTerms() { + return terms; + } + + /** + * Retruns the field + * @return the field + */ + public String getField() { + return this.field; + } + + /** + * Returns the frequency of the given term in the field or null if the term + * doesn't exist in this field. + * + * @param text + * the term to lookup + * @return the frequency of the given term in the field or null if the term + * doesn't exist in this field. + * @throws IOException + * if an {@link IOException} occurs. + */ + public final long freq(BytesRef text) throws IOException { + termsEnum = termsEnum == null ? terms.iterator(null) : termsEnum; + if (termsEnum.seekExact(text, true)) { + return freq(termsEnum); + } else { + return 0; + } + } + + /** + * Returns the total term frequency in the field. + * + * @return the total term frequency in the field. + * @throws IOException + * if an {@link IOException} occurs. + */ + public final long totalFreq() throws IOException { + return totalFreq; + } + + /** + * Returns the frequency for the term the given {@link TermsEnum} points to + * according to the {@link FieldStatistics} implemenation. + * + * @param termsEnum + * the {@link TermsEnum} instance to get the statistics from. + * @return the frequency for the term the given {@link TermsEnum} points to. + * @throws IOException + * if an {@link IOException} occurs. + */ + protected abstract long freq(TermsEnum termsEnum) throws IOException; + + } + + /** + * Document based {@link FieldStatistics} using {@link Terms#getDocCount()} if + * available or {@link IndexReader#maxDoc()} for total counts and + * {@link TermsEnum#docFreq()} for term frequencies. + */ + public static final class DocFreqFieldStatistics extends FieldStatistics { + + /** + * Creates a new {@link DocFreqFieldStatistics} instance + * @param terms the {@link Terms} instance to get the stats from + * @param field this {@link FieldStatistics} field + * @param reader the reader to obtain a fallback maxDoc count from + * @throws IOException if an {@link IOException} occurs. + */ + public DocFreqFieldStatistics(Terms terms, String field, IndexReader reader) throws IOException { + super(terms, field, terms.getDocCount() == -1 ? reader.maxDoc() : terms.getDocCount()); + } + + @Override + public long freq(TermsEnum termsEnum) throws IOException { + return termsEnum.docFreq(); + } + } + + /** + * Total Term Frequency based {@link FieldStatistics} using + * {@link Terms#getSumTotalTermFreq()} for total counts and + * {@link TermsEnum#totalTermFreq()} for term frequencies. + */ + public static final class TotalTermFreqFieldStatistics extends FieldStatistics { + /** + * Creates a new {@link TotalTermFreqFieldStatistics} instance + * @param terms the {@link Terms} instance to get the stats from + * @param field this {@link FieldStatistics} field + * @throws IOException if an {@link IOException} occurs. + */ + public TotalTermFreqFieldStatistics(Terms terms, String field) throws IOException { + super(terms, field, terms.getSumTotalTermFreq()); + if (totalFreq() == -1) { + throw new IllegalArgumentException("Terms doesn't support totalTermFreq / sumTotalTermFreq"); + } + } + + @Override + public long freq(TermsEnum termsEnum) throws IOException { + assert termsEnum.totalTermFreq() != -1; + return termsEnum.totalTermFreq(); + } + } } Index: lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (revision 1453392) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.spell.SuggestMode; -import org.apache.lucene.util.BytesRef; /** *

@@ -228,7 +227,7 @@ origIndexes[k] = i + k; } SuggestWord word = new SuggestWord(); - word.freq = combinedTermFreq; + word.docFreq = combinedTermFreq; word.score = origIndexes.length - 1; word.string = combinedTerm.text(); CombineSuggestionWrapper suggestion = new CombineSuggestionWrapper( @@ -277,9 +276,9 @@ String rightText = termText.substring(end); SuggestWord leftWord = generateSuggestWord(ir, term.field(), leftText); - if (leftWord.freq >= useMinSuggestionFrequency) { + if (leftWord.docFreq >= useMinSuggestionFrequency) { SuggestWord rightWord = generateSuggestWord(ir, term.field(), rightText); - if (rightWord.freq >= useMinSuggestionFrequency) { + if (rightWord.docFreq >= useMinSuggestionFrequency) { SuggestWordArrayWrapper suggestion = new SuggestWordArrayWrapper( newSuggestion(prefix, leftWord, rightWord)); suggestions.offer(suggestion); @@ -320,7 +319,7 @@ for (int i = 0; i < prefix.length; i++) { SuggestWord word = new SuggestWord(); word.string = prefix[i].string; - word.freq = prefix[i].freq; + word.docFreq = prefix[i].docFreq; word.score = score; newSuggestion[i] = word; } @@ -335,7 +334,7 @@ Term term = new Term(fieldname, text); int freq = ir.docFreq(term); SuggestWord word = new SuggestWord(); - word.freq = freq; + word.docFreq = freq; word.score = 1; word.string = text; return word; @@ -478,9 +477,9 @@ if (o1.numCombinations != o2.numCombinations) { return o2.numCombinations - o1.numCombinations; } - if (o1.combineSuggestion.suggestion.freq != o2.combineSuggestion.suggestion.freq) { - return o1.combineSuggestion.suggestion.freq - - o2.combineSuggestion.suggestion.freq; + if (o1.combineSuggestion.suggestion.docFreq != o2.combineSuggestion.suggestion.docFreq) { + return o1.combineSuggestion.suggestion.docFreq + - o2.combineSuggestion.suggestion.docFreq; } return 0; } @@ -496,8 +495,8 @@ int aFreqSum = 0; int aFreqMax = 0; for (SuggestWord sw : suggestWords) { - aFreqSum += sw.freq; - aFreqMax = Math.max(aFreqMax, sw.freq); + aFreqSum += sw.docFreq; + aFreqMax = Math.max(aFreqMax, sw.docFreq); } this.freqSum = aFreqSum; this.freqMax = aFreqMax; Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java (working copy) @@ -38,9 +38,14 @@ public float score; /** - * The freq of the word + * The total term frequency of the word */ - public int freq; + public long totalTermFreq; + + /** + * The document frequency of the word + */ + public int docFreq; /** * the suggested word Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java (working copy) @@ -26,7 +26,7 @@ public class SuggestWordFrequencyComparator implements Comparator { /** - * Creates a new comparator that will compare by {@link SuggestWord#freq}, + * Creates a new comparator that will compare by {@link SuggestWord#docFreq}, * then by {@link SuggestWord#score}, then by {@link SuggestWord#string}. */ public SuggestWordFrequencyComparator() {} @@ -34,10 +34,10 @@ @Override public int compare(SuggestWord first, SuggestWord second) { // first criteria: the frequency - if (first.freq > second.freq) { + if (first.docFreq > second.docFreq) { return 1; } - if (first.freq < second.freq) { + if (first.docFreq < second.docFreq) { return -1; } Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java (working copy) @@ -27,7 +27,7 @@ /** * Creates a new comparator that will compare by {@link SuggestWord#score}, - * then by {@link SuggestWord#freq}, then by {@link SuggestWord#string}. + * then by {@link SuggestWord#docFreq}, then by {@link SuggestWord#string}. */ public SuggestWordScoreComparator() {} @@ -40,13 +40,20 @@ if (first.score < second.score) { return -1; } - + // second criteria (if first criteria is equal): the popularity - if (first.freq > second.freq) { + if (first.totalTermFreq < second.totalTermFreq) { return 1; } + if (first.totalTermFreq > second.totalTermFreq) { + return -1; + } + + if (first.docFreq > second.docFreq) { + return 1; + } - if (first.freq < second.freq) { + if (first.docFreq < second.docFreq) { return -1; } // third criteria: term text Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (working copy) @@ -391,9 +391,9 @@ } if (ir != null && field != null) { // use the user index - sugWord.freq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index + sugWord.docFreq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index // don't suggest a word that is not present in the field - if ((suggestMode==SuggestMode.SUGGEST_MORE_POPULAR && goalFreq > sugWord.freq) || sugWord.freq < 1) { + if ((suggestMode==SuggestMode.SUGGEST_MORE_POPULAR && goalFreq > sugWord.docFreq) || sugWord.docFreq < 1) { continue; } } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (revision 1453360) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (working copy) @@ -175,10 +175,7 @@ @SuppressWarnings("unchecked") public List applyAffix(char strippedWord[], int length, HunspellAffix affix, int recursionDepth) { if(dictionary.isIgnoreCase()) { - for(int i=0;i= limit; + assert offset <=0 && offset <= buffer.length; + for (int i = offset; i < limit;) { + i += Character.toChars( + Character.toLowerCase( + codePointAt(buffer, i)), buffer, i); + } + } + + /** * Fills the {@link CharacterBuffer} with characters read from the given * reader {@link Reader}. This method tries to read as many characters into * the {@link CharacterBuffer} as possible, each call to fill will start Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java (revision 1453360) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharArrayMap.java (working copy) @@ -215,12 +215,9 @@ * The user should never modify this text array after calling this method. */ public V put(char[] text, V value) { - if (ignoreCase) - for(int i=0;i= term.text().length()) { // can only match if it's exact return new SingleTermsEnum(terms.iterator(null), term.bytes()); } - return new FuzzyTermsEnum(terms, atts, getTerm(), maxEdits, prefixLength, transpositions); + return new FuzzyTermsEnum(terms, atts, getTerm().text(), maxEdits, prefixLength, transpositions); } /** Index: lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1453360) +++ lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -74,7 +74,7 @@ protected final boolean raw; protected final Terms terms; - private final Term term; + private final String term; protected final int termText[]; protected final int realPrefixLength; @@ -98,7 +98,7 @@ * @param prefixLength Length of required common prefix. Default value is 0. * @throws IOException if there is a low-level IO error */ - public FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, + public FuzzyTermsEnum(Terms terms, AttributeSource atts, final String term, final float minSimilarity, final int prefixLength, boolean transpositions) throws IOException { if (minSimilarity >= 1.0f && minSimilarity != (int)minSimilarity) throw new IllegalArgumentException("fractional edit distances are not allowed"); @@ -107,13 +107,12 @@ if(prefixLength < 0) throw new IllegalArgumentException("prefixLength cannot be less than 0"); this.terms = terms; - this.term = term; // convert the string into a utf32 int[] representation for fast comparisons - final String utf16 = term.text(); - this.termText = new int[utf16.codePointCount(0, utf16.length())]; - for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp)) - termText[j++] = cp = utf16.codePointAt(i); + this.term = term; + this.termText = new int[term.codePointCount(0, term.length())]; + for (int cp, i = 0, j = 0; i < term.length(); i += Character.charCount(cp)) + termText[j++] = cp = term.codePointAt(i); this.termLength = termText.length; this.dfaAtt = atts.addAttribute(LevenshteinAutomataAttribute.class); @@ -342,7 +341,7 @@ this.matchers = new ByteRunAutomaton[compiled.length]; for (int i = 0; i < compiled.length; i++) this.matchers[i] = compiled[i].runAutomaton; - termRef = new BytesRef(term.text()); + termRef = new BytesRef(term); } /** finds the smallest Lev(n) DFA that accepts the term. */