Index: solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/DirectSolrSpellChecker.java (working copy) @@ -203,7 +203,7 @@ } if (!foundOriginal) { SuggestWord orig = new SuggestWord(); - orig.freq = freq; + orig.docFreq = freq; orig.string = tokenText; suggestionsWithOrig[0] = orig; suggestions = suggestionsWithOrig; @@ -214,7 +214,7 @@ result.add(token, empty); } else { for (SuggestWord suggestion : suggestions) { - result.add(token, suggestion.string, suggestion.freq); + result.add(token, suggestion.string, suggestion.docFreq); } } } Index: solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/WordBreakSolrSpellChecker.java (working copy) @@ -239,9 +239,9 @@ firstOne = false; sb.append(word.string); if (sortMethod == BreakSuggestionSortMethod.NUM_CHANGES_THEN_MAX_FREQUENCY) { - freq = Math.max(freq, word.freq); + freq = Math.max(freq, word.docFreq); } else { - freq += word.freq; + freq += word.docFreq; } } breakSuggestionList.add(new ResultEntry(tokenArr[i], sb.toString(), @@ -268,7 +268,7 @@ Token token = new Token(sb.toString(), tokenArr[firstTermIndex] .startOffset(), tokenArr[lastTermIndex].endOffset()); combineSuggestionList.add(new ResultEntry(token, cs.suggestion.string, - cs.suggestion.freq)); + cs.suggestion.docFreq)); } } Index: solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/spelling/SolrSpellChecker.java (working copy) @@ -134,7 +134,7 @@ Integer o = mergeData.origVsFreq.get(original); if (o != null) result.addFrequency(token, o); for (SuggestWord word : suggestions) - result.add(token, word.string, word.freq); + result.add(token, word.string, word.docFreq); } else { List words = new ArrayList(sugQueue.size()); for (SuggestWord word : suggestions) words.add(word.string); Index: solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (revision 1453360) +++ solr/core/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (working copy) @@ -378,7 +378,7 @@ if (suggestion.getAlternativeFrequencies() != null && suggestion.getAlternativeFrequencies().size() > 0) { Integer freq = suggestion.getAlternativeFrequencies().get(i); - if (freq != null) sug.freq += freq; + if (freq != null) sug.docFreq += freq; } } } Index: lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java =================================================================== --- lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java (revision 1453360) +++ lucene/suggest/src/test/org/apache/lucene/search/spell/TestWordBreakSpellChecker.java (working copy) @@ -210,8 +210,8 @@ Assert.assertTrue(sw[0][1].string.equals("thousand")); Assert.assertTrue(sw[0][0].score == 1); Assert.assertTrue(sw[0][1].score == 1); - Assert.assertTrue(sw[0][1].freq>1); - Assert.assertTrue(sw[0][0].freq>sw[0][1].freq); + Assert.assertTrue(sw[0][1].docFreq>1); + Assert.assertTrue(sw[0][0].docFreq>sw[0][1].docFreq); Assert.assertTrue(sw[1].length==3); Assert.assertTrue(sw[1][0].string.equals("one")); Assert.assertTrue(sw[1][1].string.equals("thou")); @@ -219,9 +219,9 @@ Assert.assertTrue(sw[1][0].score == 2); Assert.assertTrue(sw[1][1].score == 2); Assert.assertTrue(sw[1][2].score == 2); - Assert.assertTrue(sw[1][0].freq>1); - Assert.assertTrue(sw[1][1].freq==1); - Assert.assertTrue(sw[1][2].freq==1); + Assert.assertTrue(sw[1][0].docFreq>1); + Assert.assertTrue(sw[1][1].docFreq==1); + Assert.assertTrue(sw[1][2].docFreq==1); } { Term term = new Term("numbers", "onethousandonehundredeleven"); Index: lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/DirectSpellChecker.java (working copy) @@ -19,8 +19,10 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BoostAttribute; import org.apache.lucene.search.FuzzyTermsEnum; import org.apache.lucene.search.MaxNonCompetitiveBoostAttribute; @@ -88,6 +90,7 @@ /** the string distance to use */ private StringDistance distance = INTERNAL_LEVENSHTEIN; + /** Creates a DirectSpellChecker with default configuration values */ public DirectSpellChecker() {} @@ -317,54 +320,85 @@ */ public SuggestWord[] suggestSimilar(Term term, int numSug, IndexReader ir, SuggestMode suggestMode, float accuracy) throws IOException { - final CharsRef spare = new CharsRef(); - String text = term.text(); - if (minQueryLength > 0 && text.codePointCount(0, text.length()) < minQueryLength) + FieldStatistics stats = getFieldStatistics(ir, term.field()); + if (stats == null) { + return new SuggestWord[0]; + } + final CharsRef spare = new CharsRef(); + return suggestSimilar(term.bytes(), numSug, stats, suggestMode, accuracy, spare); + } + + /** + * Suggest similar words. + * + *

Unlike {@link SpellChecker}, the similarity used to fetch the most + * relevant terms is an edit distance, therefore typically a low value + * for numSug will work very well. + * + * @param utf8Text the term you want to spell check on + * @param numSug the maximum number of suggested words + * @param stats the {@link FieldStatistics} to use to find the relevant terms + * @param suggestMode specifies when to return suggested words + * @param accuracy return only suggested words that match with this similarity + * @return sorted list of the suggested words according to the comparator + * @throws IOException If there is a low-level I/O error. + */ + public SuggestWord[] suggestSimilar(BytesRef utf8Text, int numSug, final FieldStatistics stats, + SuggestMode suggestMode, float accuracy, CharsRef spare) throws IOException { + UnicodeUtil.UTF8toUTF16(utf8Text, spare); + if (minQueryLength > 0 && Character.codePointCount(spare, 0, spare.length) < minQueryLength) return new SuggestWord[0]; if (lowerCaseTerms) { - term = new Term(term.field(), text.toLowerCase(Locale.ROOT)); - } + final char[] buffer = spare.chars; + final int length = spare.length; + for (int i = 0; i < length;) { + i += Character.toChars( + Character.toLowerCase( + Character.codePointAt(buffer, i)), buffer, i); + } + utf8Text = new BytesRef(spare); + } + final String utf16Term = spare.toString(); + long termFrequency = stats.freq(utf8Text); - int docfreq = ir.docFreq(term); - - if (suggestMode==SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && docfreq > 0) { + if (suggestMode==SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX && termFrequency > 0) { return new SuggestWord[0]; } - int maxDoc = ir.maxDoc(); + long totalFrequency = stats.totalFreq(); - if (maxQueryFrequency >= 1f && docfreq > maxQueryFrequency) { + if (maxQueryFrequency >= 1f && termFrequency > maxQueryFrequency) { return new SuggestWord[0]; - } else if (docfreq > (int) Math.ceil(maxQueryFrequency * (float)maxDoc)) { + } else if (termFrequency > (long) Math.ceil(maxQueryFrequency * (double)totalFrequency)) { return new SuggestWord[0]; } - if (suggestMode!=SuggestMode.SUGGEST_MORE_POPULAR) docfreq = 0; + if (suggestMode!=SuggestMode.SUGGEST_MORE_POPULAR) termFrequency = 0; if (thresholdFrequency >= 1f) { - docfreq = Math.max(docfreq, (int) thresholdFrequency); + termFrequency = Math.max(termFrequency, (long) thresholdFrequency); } else if (thresholdFrequency > 0f) { - docfreq = Math.max(docfreq, (int)(thresholdFrequency * (float)maxDoc)-1); + termFrequency = Math.max(termFrequency, (long)(thresholdFrequency * (double)totalFrequency)-1); } - Collection terms = null; + Collection scoreTerms = null; int inspections = numSug * maxInspections; // try ed=1 first, in case we get lucky - terms = suggestSimilar(term, inspections, ir, docfreq, 1, accuracy, spare); - if (maxEdits > 1 && terms.size() < inspections) { + scoreTerms = suggestSimilar(utf8Text, utf16Term, inspections, stats, termFrequency, 1, accuracy, spare); + if (maxEdits > 1 && scoreTerms.size() < inspections) { HashSet moreTerms = new HashSet(); - moreTerms.addAll(terms); - moreTerms.addAll(suggestSimilar(term, inspections, ir, docfreq, maxEdits, accuracy, spare)); - terms = moreTerms; + moreTerms.addAll(scoreTerms); + moreTerms.addAll(suggestSimilar(utf8Text, utf16Term, inspections, stats, termFrequency, maxEdits, accuracy, spare)); + scoreTerms = moreTerms; } // create the suggestword response, sort it, and trim it to size. - SuggestWord suggestions[] = new SuggestWord[terms.size()]; + SuggestWord suggestions[] = new SuggestWord[scoreTerms.size()]; int index = suggestions.length - 1; - for (ScoreTerm s : terms) { + for (ScoreTerm s : scoreTerms) { SuggestWord suggestion = new SuggestWord(); if (s.termAsString == null) { UnicodeUtil.UTF8toUTF16(s.term, spare); @@ -372,7 +406,8 @@ } suggestion.string = s.termAsString; suggestion.score = s.score; - suggestion.freq = s.docfreq; + suggestion.totalTermFreq = s.totalTermFreq; + suggestion.docFreq = s.docfreq; suggestions[index--] = suggestion; } @@ -384,34 +419,33 @@ } return suggestions; } - /** * Provide spelling corrections based on several parameters. * - * @param term The term to suggest spelling corrections for + * @param utf8Term The UTF-8 term to suggest spelling corrections for + * @param utf16Term The UTF-16 term to suggest spelling corrections for * @param numSug The maximum number of spelling corrections - * @param ir The index reader to fetch the candidate spelling corrections from - * @param docfreq The minimum document frequency a potential suggestion need to have in order to be included + * @param stats The statistics for the field to fetch corrections from + * @param totalFrequency The minimum total term frequency a potential suggestion need to have in order to be included * @param editDistance The maximum edit distance candidates are allowed to have * @param accuracy The minimum accuracy a suggested spelling correction needs to have in order to be included * @param spare a chars scratch * @return a collection of spelling corrections sorted by ScoreTerm's natural order. * @throws IOException If I/O related errors occur */ - protected Collection suggestSimilar(Term term, int numSug, IndexReader ir, int docfreq, int editDistance, + protected Collection suggestSimilar(BytesRef utf8Term, String utf16Term, int numSug, FieldStatistics stats, long totalFrequency, int editDistance, float accuracy, final CharsRef spare) throws IOException { AttributeSource atts = new AttributeSource(); MaxNonCompetitiveBoostAttribute maxBoostAtt = atts.addAttribute(MaxNonCompetitiveBoostAttribute.class); - Terms terms = MultiFields.getTerms(ir, term.field()); + final Terms terms = stats.getTerms(); if (terms == null) { return Collections.emptyList(); } - FuzzyTermsEnum e = new FuzzyTermsEnum(terms, atts, term, editDistance, Math.max(minPrefix, editDistance-1), true); + FuzzyTermsEnum e = new FuzzyTermsEnum(stats.getTerms(), atts, utf16Term, editDistance, Math.max(minPrefix, editDistance-1), true); final PriorityQueue stQueue = new PriorityQueue(); - BytesRef queryTerm = new BytesRef(term.text()); BytesRef candidateTerm; ScoreTerm st = new ScoreTerm(); BoostAttribute boostAtt = @@ -423,15 +457,13 @@ continue; // ignore exact match of the same term - if (queryTerm.bytesEquals(candidateTerm)) + if (utf8Term.bytesEquals(candidateTerm)) continue; - int df = e.docFreq(); - - // check docFreq if required - if (df <= docfreq) + + // check ttf if required + if (stats.freq(e) <= totalFrequency) continue; - final float score; final String termAsString; if (distance == INTERNAL_LEVENSHTEIN) { @@ -442,7 +474,7 @@ } else { UnicodeUtil.UTF8toUTF16(candidateTerm, spare); termAsString = spare.toString(); - score = distance.getDistance(term.text(), termAsString); + score = distance.getDistance(utf16Term, termAsString); } if (score < accuracy) @@ -451,7 +483,8 @@ // add new entry in PQ st.term = BytesRef.deepCopyOf(candidateTerm); st.boost = boost; - st.docfreq = df; + st.totalTermFreq = e.totalTermFreq();; + st.docfreq = e.docFreq(); st.termAsString = termAsString; st.score = score; stQueue.offer(st); @@ -482,6 +515,11 @@ * The df of the spellcheck correction. */ public int docfreq; + + /** + * The total term frequency of the spellcheck correction + */ + public long totalTermFreq; /** * The spellcheck correction represented as string, can be null. @@ -529,4 +567,131 @@ return true; } } + + /** + * A simple wrapper around at {@link Terms} instance providing syntactic sugar + * to access term frequency statistics per fiedl. + * @lucene.internal + */ + public abstract static class FieldStatistics { + + private final Terms terms; + private final long totalFreq; + private TermsEnum termsEnum; + private final String field; + + /** + * Creates a new {@link FieldStatistics} instance + * @param terms the {@link Terms} to wrap + * @param field the field + * @param totalFrequency the total frequency for this field + */ + public FieldStatistics(Terms terms, String field, long totalFrequency) { + this.terms = terms; + this.totalFreq = totalFrequency; + this.field = field; + } + + /** + * Returns the {@link Terms} instance for this field. + * @return the {@link Terms} instance for this field. + */ + public Terms getTerms() { + return terms; + } + + /** + * Retruns the field + * @return the field + */ + public String getField() { + return this.field; + } + + /** + * Returns the frequency of the given term in the field or null if the term + * doesn't exist in this field. + * + * @param text + * the term to lookup + * @return the frequency of the given term in the field or null if the term + * doesn't exist in this field. + * @throws IOException + * if an {@link IOException} occurs. + */ + public final long freq(BytesRef text) throws IOException { + termsEnum = termsEnum == null ? terms.iterator(null) : termsEnum; + if (termsEnum.seekExact(text, true)) { + return freq(termsEnum); + } else { + return 0; + } + } + + /** + * Returns the total term frequency in the field. + * + * @return the total term frequency in the field. + * @throws IOException + * if an {@link IOException} occurs. + */ + public final long totalFreq() throws IOException { + return totalFreq; + } + + /** + * Returns the frequency for the term the given {@link TermsEnum} points to + * according to the {@link FieldStatistics} implemenation. + * + * @param termsEnum + * the {@link TermsEnum} instance to get the statistics from. + * @return the frequency for the term the given {@link TermsEnum} points to. + * @throws IOException + * if an {@link IOException} occurs. + */ + protected abstract long freq(TermsEnum termsEnum) throws IOException; + + } + + /** + * Returns {@link FieldStatistics} for the given field and reader. If the + * field supports {@link Terms#getSumTotalTermFreq()} the + * {@link FieldStatistics} will use total term frequency rather than document + * frequency as the terms statistics. + * + * @param reader + * the reader to build the statistics from. + * @param field + * the field to build the statistics for. + * @return a new {@link FieldStatistics} instance for the given field or + * null if the field has no {@link Terms}. + * @throws IOException + * if an {@link IOException} occurs + */ + public static FieldStatistics getFieldStatistics(IndexReader reader, String field) throws IOException { + final Terms terms = MultiFields.getTerms(reader, field); + if (terms == null) { + return null; + } + final long sumTotalTermFreq = terms.getSumTotalTermFreq(); + if (sumTotalTermFreq == -1) { + int docCount = terms.getDocCount(); + // fall back to max doc if we don't have docCount + return new FieldStatistics(terms, field, docCount == -1 ? reader.maxDoc() : docCount) { + @Override + public long freq(TermsEnum termsEnum) throws IOException { + return termsEnum.docFreq(); + } + }; + } + + return new FieldStatistics(terms, field, sumTotalTermFreq) { + @Override + public long freq(TermsEnum termsEnum) throws IOException { + assert termsEnum.totalTermFreq() != -1; + return termsEnum.totalTermFreq(); + } + }; + + } } Index: lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (revision 1453392) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/WordBreakSpellChecker.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.spell.SuggestMode; -import org.apache.lucene.util.BytesRef; /** *

@@ -228,7 +227,7 @@ origIndexes[k] = i + k; } SuggestWord word = new SuggestWord(); - word.freq = combinedTermFreq; + word.docFreq = combinedTermFreq; word.score = origIndexes.length - 1; word.string = combinedTerm.text(); CombineSuggestionWrapper suggestion = new CombineSuggestionWrapper( @@ -277,9 +276,9 @@ String rightText = termText.substring(end); SuggestWord leftWord = generateSuggestWord(ir, term.field(), leftText); - if (leftWord.freq >= useMinSuggestionFrequency) { + if (leftWord.docFreq >= useMinSuggestionFrequency) { SuggestWord rightWord = generateSuggestWord(ir, term.field(), rightText); - if (rightWord.freq >= useMinSuggestionFrequency) { + if (rightWord.docFreq >= useMinSuggestionFrequency) { SuggestWordArrayWrapper suggestion = new SuggestWordArrayWrapper( newSuggestion(prefix, leftWord, rightWord)); suggestions.offer(suggestion); @@ -320,7 +319,7 @@ for (int i = 0; i < prefix.length; i++) { SuggestWord word = new SuggestWord(); word.string = prefix[i].string; - word.freq = prefix[i].freq; + word.docFreq = prefix[i].docFreq; word.score = score; newSuggestion[i] = word; } @@ -335,7 +334,7 @@ Term term = new Term(fieldname, text); int freq = ir.docFreq(term); SuggestWord word = new SuggestWord(); - word.freq = freq; + word.docFreq = freq; word.score = 1; word.string = text; return word; @@ -478,9 +477,9 @@ if (o1.numCombinations != o2.numCombinations) { return o2.numCombinations - o1.numCombinations; } - if (o1.combineSuggestion.suggestion.freq != o2.combineSuggestion.suggestion.freq) { - return o1.combineSuggestion.suggestion.freq - - o2.combineSuggestion.suggestion.freq; + if (o1.combineSuggestion.suggestion.docFreq != o2.combineSuggestion.suggestion.docFreq) { + return o1.combineSuggestion.suggestion.docFreq + - o2.combineSuggestion.suggestion.docFreq; } return 0; } @@ -496,8 +495,8 @@ int aFreqSum = 0; int aFreqMax = 0; for (SuggestWord sw : suggestWords) { - aFreqSum += sw.freq; - aFreqMax = Math.max(aFreqMax, sw.freq); + aFreqSum += sw.docFreq; + aFreqMax = Math.max(aFreqMax, sw.docFreq); } this.freqSum = aFreqSum; this.freqMax = aFreqMax; Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWord.java (working copy) @@ -38,9 +38,14 @@ public float score; /** - * The freq of the word + * The total term frequency of the word */ - public int freq; + public long totalTermFreq; + + /** + * The document frequency of the word + */ + public int docFreq; /** * the suggested word Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordFrequencyComparator.java (working copy) @@ -26,7 +26,7 @@ public class SuggestWordFrequencyComparator implements Comparator { /** - * Creates a new comparator that will compare by {@link SuggestWord#freq}, + * Creates a new comparator that will compare by {@link SuggestWord#docFreq}, * then by {@link SuggestWord#score}, then by {@link SuggestWord#string}. */ public SuggestWordFrequencyComparator() {} @@ -34,10 +34,10 @@ @Override public int compare(SuggestWord first, SuggestWord second) { // first criteria: the frequency - if (first.freq > second.freq) { + if (first.docFreq > second.docFreq) { return 1; } - if (first.freq < second.freq) { + if (first.docFreq < second.docFreq) { return -1; } Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SuggestWordScoreComparator.java (working copy) @@ -27,7 +27,7 @@ /** * Creates a new comparator that will compare by {@link SuggestWord#score}, - * then by {@link SuggestWord#freq}, then by {@link SuggestWord#string}. + * then by {@link SuggestWord#docFreq}, then by {@link SuggestWord#string}. */ public SuggestWordScoreComparator() {} @@ -40,13 +40,20 @@ if (first.score < second.score) { return -1; } - + // second criteria (if first criteria is equal): the popularity - if (first.freq > second.freq) { + if (first.totalTermFreq < second.totalTermFreq) { return 1; } + if (first.totalTermFreq > second.totalTermFreq) { + return -1; + } + + if (first.docFreq > second.docFreq) { + return 1; + } - if (first.freq < second.freq) { + if (first.docFreq < second.docFreq) { return -1; } // third criteria: term text Index: lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (revision 1453360) +++ lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (working copy) @@ -391,9 +391,9 @@ } if (ir != null && field != null) { // use the user index - sugWord.freq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index + sugWord.docFreq = ir.docFreq(new Term(field, sugWord.string)); // freq in the index // don't suggest a word that is not present in the field - if ((suggestMode==SuggestMode.SUGGEST_MORE_POPULAR && goalFreq > sugWord.freq) || sugWord.freq < 1) { + if ((suggestMode==SuggestMode.SUGGEST_MORE_POPULAR && goalFreq > sugWord.docFreq) || sugWord.docFreq < 1) { continue; } } Index: lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (revision 1453360) +++ lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (working copy) @@ -211,7 +211,7 @@ AttributeSource atts = new AttributeSource(); MaxNonCompetitiveBoostAttribute maxBoostAtt = atts.addAttribute(MaxNonCompetitiveBoostAttribute.class); - SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength); + SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm.text(), f.minSimilarity, f.prefixLength); //store the df so all variants use same idf int df = reader.docFreq(startTerm); int numVariants = 0; Index: lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java (revision 1453360) +++ lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyQuery.java (working copy) @@ -146,7 +146,7 @@ if (!termLongEnough) { // can only match if it's exact return new SingleTermsEnum(terms.iterator(null), term.bytes()); } - return new SlowFuzzyTermsEnum(terms, atts, getTerm(), minimumSimilarity, prefixLength); + return new SlowFuzzyTermsEnum(terms, atts, getTerm().text(), minimumSimilarity, prefixLength); } /** Index: lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (revision 1453360) +++ lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (working copy) @@ -43,7 +43,7 @@ @Deprecated public final class SlowFuzzyTermsEnum extends FuzzyTermsEnum { - public SlowFuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, + public SlowFuzzyTermsEnum(Terms terms, AttributeSource atts, String term, float minSimilarity, int prefixLength) throws IOException { super(terms, atts, term, minSimilarity, prefixLength, false); } Index: lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java (revision 1453360) +++ lucene/core/src/java/org/apache/lucene/search/FuzzyQuery.java (working copy) @@ -137,7 +137,7 @@ if (maxEdits == 0 || prefixLength >= term.text().length()) { // can only match if it's exact return new SingleTermsEnum(terms.iterator(null), term.bytes()); } - return new FuzzyTermsEnum(terms, atts, getTerm(), maxEdits, prefixLength, transpositions); + return new FuzzyTermsEnum(terms, atts, getTerm().text(), maxEdits, prefixLength, transpositions); } /** Index: lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1453360) +++ lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -74,7 +74,7 @@ protected final boolean raw; protected final Terms terms; - private final Term term; + private final String term; protected final int termText[]; protected final int realPrefixLength; @@ -98,7 +98,7 @@ * @param prefixLength Length of required common prefix. Default value is 0. * @throws IOException if there is a low-level IO error */ - public FuzzyTermsEnum(Terms terms, AttributeSource atts, Term term, + public FuzzyTermsEnum(Terms terms, AttributeSource atts, final String term, final float minSimilarity, final int prefixLength, boolean transpositions) throws IOException { if (minSimilarity >= 1.0f && minSimilarity != (int)minSimilarity) throw new IllegalArgumentException("fractional edit distances are not allowed"); @@ -107,13 +107,12 @@ if(prefixLength < 0) throw new IllegalArgumentException("prefixLength cannot be less than 0"); this.terms = terms; - this.term = term; // convert the string into a utf32 int[] representation for fast comparisons - final String utf16 = term.text(); - this.termText = new int[utf16.codePointCount(0, utf16.length())]; - for (int cp, i = 0, j = 0; i < utf16.length(); i += Character.charCount(cp)) - termText[j++] = cp = utf16.codePointAt(i); + this.term = term; + this.termText = new int[term.codePointCount(0, term.length())]; + for (int cp, i = 0, j = 0; i < term.length(); i += Character.charCount(cp)) + termText[j++] = cp = term.codePointAt(i); this.termLength = termText.length; this.dfaAtt = atts.addAttribute(LevenshteinAutomataAttribute.class); @@ -342,7 +341,7 @@ this.matchers = new ByteRunAutomaton[compiled.length]; for (int i = 0; i < compiled.length; i++) this.matchers[i] = compiled[i].runAutomaton; - termRef = new BytesRef(term.text()); + termRef = new BytesRef(term); } /** finds the smallest Lev(n) DFA that accepts the term. */