Index: lucene/common-build.xml =================================================================== --- lucene/common-build.xml (revision 1298343) +++ lucene/common-build.xml (working copy) @@ -644,7 +644,8 @@ - + + Index: lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java (revision 1298343) +++ lucene/core/src/java/org/apache/lucene/util/UnicodeUtil.java (working copy) @@ -510,7 +510,7 @@ for (int r = offset, e = offset + count; r < e; ++r) { int cp = codePoints[r]; if (cp < 0 || cp > 0x10ffff) { - throw new IllegalArgumentException(); + throw new IllegalArgumentException("cp=" + cp); } while (true) { try { Index: lucene/core/src/java/org/apache/lucene/util/fst/FST.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/FST.java (revision 1298343) +++ lucene/core/src/java/org/apache/lucene/util/fst/FST.java (working copy) @@ -165,7 +165,7 @@ // From node (ord or address); currently only used when // building an FST w/ willPackFST=true: - int node; + public int node; // To node (ord or address): public int target; @@ -816,7 +816,7 @@ * * @return Returns the second argument (arc). */ - public Arc readFirstTargetArc(Arc follow, Arc arc) throws IOException { + public Arc readFirstTargetArc(Arc follow, Arc arc, BytesReader in) throws IOException { //int pos = address; //System.out.println(" readFirstTarget follow.target=" + follow.target + " isFinal=" + follow.isFinal()); if (follow.isFinal()) { @@ -835,7 +835,7 @@ //System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output)); return arc; } else { - return readFirstRealTargetArc(follow.target, arc, getBytesReader(0)); + return readFirstRealTargetArc(follow.target, arc, in); } } @@ -885,15 +885,15 @@ } /** In-place read; returns the arc. */ - public Arc readNextArc(Arc arc) throws IOException { + public Arc readNextArc(Arc arc, BytesReader in) throws IOException { if (arc.label == END_LABEL) { // This was a fake inserted "final" arc if (arc.nextArc <= 0) { throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true"); } - return readFirstRealTargetArc(arc.nextArc, arc, getBytesReader(0)); + return readFirstRealTargetArc(arc.nextArc, arc, in); } else { - return readNextRealArc(arc, getBytesReader(0)); + return readNextRealArc(arc, in); } } Index: lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java (revision 1298343) +++ lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java (working copy) @@ -36,6 +36,7 @@ protected final T NO_OUTPUT; protected final FST.Arc scratchArc = new FST.Arc(); + protected final FST.BytesReader fstReader; protected int upto; protected int targetLength; @@ -45,6 +46,7 @@ * term before target. */ protected FSTEnum(FST fst) { this.fst = fst; + fstReader = fst.getBytesReader(0); NO_OUTPUT = fst.outputs.getNoOutput(); fst.getFirstArc(getArc(0)); output[0] = NO_OUTPUT; @@ -62,7 +64,7 @@ if (upto == 0) { //System.out.println(" init"); upto = 1; - fst.readFirstTargetArc(getArc(0), getArc(1)); + fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); return; } //System.out.println(" rewind upto=" + upto + " vs targetLength=" + targetLength); @@ -78,7 +80,7 @@ } else if (cmp > 0) { // seek backwards -- reset this arc to the first arc final FST.Arc arc = getArc(upto); - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); //System.out.println(" seek first arc"); break; } @@ -92,7 +94,7 @@ if (upto == 0) { //System.out.println(" init"); upto = 1; - fst.readFirstTargetArc(getArc(0), getArc(1)); + fst.readFirstTargetArc(getArc(0), getArc(1), fstReader); } else { // pop //System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast()); @@ -103,7 +105,7 @@ return; } } - fst.readNextArc(arcs[upto]); + fst.readNextArc(arcs[upto], fstReader); } pushFirst(); @@ -180,7 +182,7 @@ } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); continue; } else if (low == arc.numArcs) { @@ -198,7 +200,7 @@ final FST.Arc prevArc = getArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.isLast()) { - fst.readNextArc(prevArc); + fst.readNextArc(prevArc, fstReader); pushFirst(); return; } @@ -221,7 +223,7 @@ } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); } else if (arc.label > targetLabel) { pushFirst(); @@ -237,7 +239,7 @@ final FST.Arc prevArc = getArc(upto); //System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast()); if (!prevArc.isLast()) { - fst.readNextArc(prevArc); + fst.readNextArc(prevArc, fstReader); pushFirst(); return; } @@ -246,7 +248,7 @@ } else { // keep scanning //System.out.println(" next scan"); - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } } } @@ -320,7 +322,7 @@ } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); continue; } else if (high == -1) { @@ -333,12 +335,12 @@ while(true) { // First, walk backwards until we find a first arc // that's before our target label: - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); if (arc.label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) { - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } pushLast(); return; @@ -370,7 +372,7 @@ } setCurrentLabel(arc.label); incr(); - arc = fst.readFirstTargetArc(arc, getArc(upto)); + arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader); targetLabel = getTargetLabel(); } else if (arc.label > targetLabel) { // TODO: if each arc could somehow read the arc just @@ -380,12 +382,12 @@ while(true) { // First, walk backwards until we find a first arc // that's before our target label: - fst.readFirstTargetArc(getArc(upto-1), arc); + fst.readFirstTargetArc(getArc(upto-1), arc, fstReader); if (arc.label < targetLabel) { // Then, scan forwards to the arc just before // the targetLabel: while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) { - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } pushLast(); return; @@ -404,7 +406,7 @@ return; } else { // keep scanning - fst.readNextArc(arc); + fst.readNextArc(arc, fstReader); } } else { pushLast(); @@ -441,7 +443,7 @@ // short circuit //upto--; //upto = 0; - fst.readFirstTargetArc(arc, getArc(upto)); + fst.readFirstTargetArc(arc, getArc(upto), fstReader); //System.out.println(" no match upto=" + upto); return false; } @@ -493,7 +495,7 @@ incr(); final FST.Arc nextArc = getArc(upto); - fst.readFirstTargetArc(arc, nextArc); + fst.readFirstTargetArc(arc, nextArc, fstReader); arc = nextArc; } } Index: lucene/core/src/java/org/apache/lucene/util/fst/Util.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/fst/Util.java (revision 1298343) +++ lucene/core/src/java/org/apache/lucene/util/fst/Util.java (working copy) @@ -335,6 +335,7 @@ final List> results = new ArrayList>(); + final FST.BytesReader fstReader = fst.getBytesReader(0); final T NO_OUTPUT = fst.outputs.getNoOutput(); // TODO: we could enable FST to sorting arcs by weight @@ -366,7 +367,7 @@ FST.Arc minArc = null; path = new FSTPath(NO_OUTPUT, fromNode, comparator); - fst.readFirstTargetArc(fromNode, path.arc); + fst.readFirstTargetArc(fromNode, path.arc, fstReader); // Bootstrap: find the min starting arc while (true) { @@ -383,7 +384,7 @@ if (path.arc.isLast()) { break; } - fst.readNextArc(path.arc); + fst.readNextArc(path.arc, fstReader); } assert minArc != null; @@ -439,7 +440,7 @@ while (true) { //System.out.println("\n cycle path: " + path); - fst.readFirstTargetArc(path.arc, path.arc); + fst.readFirstTargetArc(path.arc, path.arc, fstReader); // For each arc leaving this node: boolean foundZero = false; @@ -463,7 +464,7 @@ if (path.arc.isLast()) { break; } - fst.readNextArc(path.arc); + fst.readNextArc(path.arc, fstReader); } assert foundZero; Index: lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (revision 1298343) +++ lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (working copy) @@ -435,10 +435,10 @@ while(true) { // read all arcs: - fst.readFirstTargetArc(arc, arc); + fst.readFirstTargetArc(arc, arc, fst.getBytesReader(0)); arcs.add(new FST.Arc().copyFrom(arc)); while(!arc.isLast()) { - fst.readNextArc(arc); + fst.readNextArc(arc, fst.getBytesReader(0)); arcs.add(new FST.Arc().copyFrom(arc)); } @@ -1832,8 +1832,8 @@ throws IOException { if (FST.targetHasArcs(arc)) { int childCount = 0; - for (arc = fst.readFirstTargetArc(arc, arc);; - arc = fst.readNextArc(arc), childCount++) + for (arc = fst.readFirstTargetArc(arc, arc, fst.getBytesReader(0));; + arc = fst.readNextArc(arc, fst.getBytesReader(0)), childCount++) { boolean expanded = fst.isExpandedTarget(arc); int children = verifyStateAndBelow(fst, new FST.Arc().copyFrom(arc), depth + 1); @@ -1967,12 +1967,12 @@ assertEquals(nothing, startArc.output); assertEquals(nothing, startArc.nextFinalOutput); - FST.Arc arc = fst.readFirstTargetArc(startArc, new FST.Arc()); + FST.Arc arc = fst.readFirstTargetArc(startArc, new FST.Arc(), fst.getBytesReader(0)); assertEquals('a', arc.label); assertEquals(17, arc.nextFinalOutput.longValue()); assertTrue(arc.isFinal()); - arc = fst.readNextArc(arc); + arc = fst.readNextArc(arc, fst.getBytesReader(0)); assertEquals('b', arc.label); assertFalse(arc.isFinal()); assertEquals(42, arc.output.longValue()); Index: modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java =================================================================== --- modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java (revision 1298343) +++ modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java (working copy) @@ -135,11 +135,12 @@ try { List> rootArcs = new ArrayList>(); Arc arc = automaton.getFirstArc(new Arc()); - automaton.readFirstTargetArc(arc, arc); + final FST.BytesReader fstReader = automaton.getBytesReader(0); + automaton.readFirstTargetArc(arc, arc, fstReader); while (true) { rootArcs.add(new Arc().copyFrom(arc)); if (arc.isLast()) break; - automaton.readNextArc(arc); + automaton.readNextArc(arc, fstReader); } Collections.reverse(rootArcs); // we want highest weights first. @@ -168,13 +169,14 @@ // Get the UTF-8 bytes representation of the input key. try { final FST.Arc scratch = new FST.Arc(); + final FST.BytesReader fstReader = automaton.getBytesReader(0); for (; rootArcIndex < rootArcs.length; rootArcIndex++) { final FST.Arc rootArc = rootArcs[rootArcIndex]; final FST.Arc arc = scratch.copyFrom(rootArc); // Descend into the automaton using the key as prefix. if (descendWithPrefix(arc, utf8)) { - automaton.readFirstTargetArc(arc, arc); + automaton.readFirstTargetArc(arc, arc, fstReader); if (arc.label == FST.END_LABEL) { // Normalize prefix-encoded weight. return rootArc.label; @@ -356,8 +358,9 @@ } assert output.offset == 0; output.bytes[output.length++] = (byte) arc.label; + final FST.BytesReader fstReader = automaton.getBytesReader(0); - automaton.readFirstTargetArc(arc, arc); + automaton.readFirstTargetArc(arc, arc, fstReader); while (true) { if (arc.label == FST.END_LABEL) { res.add(new Completion(output, bucket)); @@ -373,7 +376,7 @@ if (arc.isLast()) { break; } - automaton.readNextArc(arc); + automaton.readNextArc(arc, fstReader); } return false; } Index: modules/suggest/src/java/org/apache/lucene/search/suggest/fuzzyfst/FuzzyFSTCompletionLookup.java =================================================================== --- modules/suggest/src/java/org/apache/lucene/search/suggest/fuzzyfst/FuzzyFSTCompletionLookup.java (revision 0) +++ modules/suggest/src/java/org/apache/lucene/search/suggest/fuzzyfst/FuzzyFSTCompletionLookup.java (working copy) @@ -0,0 +1,698 @@ +package org.apache.lucene.search.suggest.fuzzyfst; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; + +import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper; +import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util.fst.Builder; +import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.Outputs; +import org.apache.lucene.util.fst.PositiveIntOutputs; +import org.apache.lucene.util.fst.Util; + +// nocommit +// - hmm: once in tail, I know a given path will complete +// w/ exactly that weight (after getting first +// weight)... can we prune more aggressively? +// - eg the non-fuzzy case should use a PQ!? +// - can I prune early...? don't add a path if best freq +// is not good enough... +// - need to impl onlyMorePopular? +// - HOW to not require true maxFreq... +// - hmm: must cut over to bytes? nibbles? +// - tie breaks! +// - utf8 FST or utf32 or what! +// - int vs float...??? + +public class FuzzyFSTCompletionLookup extends Lookup { + + // nocommit: make this multiplicative, so that eg if + // correction is < 10X cost of non-completion, it's + // competitive: + + // We only allow one edit; when an edit happens, we + // penalize the score by this much. This value is added + // into the freq of the completion, so this penalty + // defines how much MORE frequent a suggestion must be for + // an edit to be allowed. The lower this is the more + // costly the search will be: + static final int EDIT_PENALTY = 1000000; + + private static boolean VERBOSE = false; + + // TODO: we can easily separate out INSERT, DELETE, + // TRANSPOSE penalties, use custom cost matrix, etc., if + // this would help... + + static String fragmentToString(IntsRef fragment) { + if (fragment.length == 0) { + return ""; + } else if (fragment.ints[fragment.offset + fragment.length-1] == FST.END_LABEL) { + return UnicodeUtil.newString(fragment.ints, fragment.offset, fragment.length-1) + ""; + } else { + return UnicodeUtil.newString(fragment.ints, fragment.offset, fragment.length); + } + } + + private class Path implements Comparable { + + // Backtrace (null for root path): + final Path back; + + // Where we are in the input text: + final int inputNode; + + // Where we are in the FST: + final FST.Arc fstNode; + + // Cost of this path: + final int cost; + + // True if this path has fuzz somewhere: + final boolean hasEdit; + + // Creation order: + final int ord; + + // If last op was a delete, this is != -1: + // nocommit can just be bool (or a bit flag) -- the arc + // then has the label: + final int priorDeleteLabel; + + public Path(int ord, Path back, int inputNode, FST.Arc fstNode, int cost, boolean hasEdit, int priorDeleteLabel) { + this.ord = ord; + this.back = back; + this.inputNode = inputNode; + this.fstNode = fstNode; + this.cost = cost; + this.hasEdit = hasEdit; + this.priorDeleteLabel = priorDeleteLabel; + } + + @Override + public int compareTo(Path other) { + if (this == other) { + return 0; + } else { + if (exactFirst) { + if (!hasEdit && other.hasEdit) { + return -1; + } else if (hasEdit && !other.hasEdit) { + return 1; + } + } + if (cost != other.cost) { + return cost - other.cost; + } else { + return ord - other.ord; + } + } + } + + @Override + public String toString() { + final IntsRef fragment = getFragment(); + final StringBuilder sb = new StringBuilder(); + sb.append(ord); + sb.append(" \""); + sb.append(fragmentToString(getFragment())); + sb.append("\" pos="); + sb.append(inputNode); + sb.append(" cost="); + sb.append(cost); + if (hasEdit) { + sb.append(" EDIT"); + } + if (priorDeleteLabel != -1) { + sb.append(" lastDel=" + (char) priorDeleteLabel); + } + return sb.toString(); + } + + public IntsRef getFragment() { + final IntsRef completion = new IntsRef(); + Path path = this; + while (path.back != null) { + if (path.fstNode != path.back.fstNode) { + completion.grow(1+completion.length); + if (true || path.fstNode.label != FST.END_LABEL) { + completion.ints[completion.length++] = path.fstNode.label; + } else { + // nocommit what...? + //assert path.back.back == null; + assert completion.length == 0; + } + } else { + // Skip insertion + assert path.hasEdit; + } + path = path.back; + } + + // Reverse: + int left = 0; + int right = completion.length-1; + while (left < right) { + final int x = completion.ints[left]; + completion.ints[left++] = completion.ints[right]; + completion.ints[right--] = x; + } + + return completion; + } + } + + public static class Completion { + public final IntsRef term; + public final int cost; + public boolean isFuzzy; + + Completion(IntsRef term, int cost, boolean isFuzzy) { + this.term = term; + this.cost = cost; + this.isFuzzy = isFuzzy; + } + + @Override + public String toString() { + String s = UnicodeUtil.newString(term.ints, term.offset, term.length) + "/" + cost; + if (isFuzzy) { + s += "/fuzzy"; + } + return s; + } + } + + private static class Queue { + private final TreeSet queue = new TreeSet(); + // nocommit maybe just dedup by hash on 1st pass...? + // then after getting all results, do better dedup...? + final Set prefixSeen = new HashSet(); + private final int targetLength; + int numPush; + + public Queue(int targetLength) { + this.targetLength = targetLength; + } + + public void add(Path path) { + if (VERBOSE) { + System.out.println(" -> path " + path); + } + numPush++; + queue.add(path); + } + + public Path pop() { + while (true) { + if (queue.isEmpty()) { + return null; + } + final Path path = queue.pollFirst(); + if (path.inputNode == targetLength) { + final IntsRef fragment = path.getFragment(); + if (prefixSeen.contains(fragment)) { + if (VERBOSE) { + System.out.println(" skip prefix " + fragmentToString(fragment) + ": already seen"); + } + //assert false; + continue; + } + if (VERBOSE) { + System.out.println(" add prefix " + fragmentToString(fragment)); + } + prefixSeen.add(fragment); + } + + return path; + } + } + } + + private FST fst; + private final int maxAnswerFreq; + + final boolean exactFirst; + + public FuzzyFSTCompletionLookup(boolean exactFirst) { + // nocommit fixme + maxAnswerFreq = 1000000000; + this.exactFirst = exactFirst; + } + + public FuzzyFSTCompletionLookup() { + this(true); + } + + public FST getFST() { + return fst; + } + + public void build(TermFreqIterator tfit) throws IOException { + BytesRef term = new BytesRef(); + TermFreqIterator iter = new FuzzyTermFreqIteratorWrapper(tfit, + BytesRef.getUTF8SortedAsUnicodeComparator()); + + // nocommit fixme + final Outputs outputs = PositiveIntOutputs.getSingleton(true); + final Builder builder = new Builder(FST.INPUT_TYPE.BYTE4, outputs); + BytesRef previous = new BytesRef(); + IntsRef scratchInts = new IntsRef(); + + while ((term = iter.next()) != null) { + final int freq = (int) iter.weight(); + if (term.compareTo(previous) != 0) { + assert freq <= maxAnswerFreq : "freq=" + freq; + UnicodeUtil.UTF8toUTF32(term, scratchInts); + builder.add(scratchInts, (long) (maxAnswerFreq - freq)); + previous.copyBytes(term); + } + } + + fst = builder.finish(); + } + + private String labelToChar(int label) { + if (label == FST.END_LABEL) { + return ""; + } else { + return "" + (char) label; + } + } + + /** Find the top count completions for the + * input fragment. This method is thread safe. */ + public List lookup(final CharSequence fragment, final int count) throws IOException { + + if (VERBOSE) { + System.out.println("\nLOOKUP: frag=" + fragment); + } + + final Queue queue = new Queue(fragment.length()); + + // We operate on code points: + // nocommit what happens if we length 1 string w/ an unpaired surrgoate...? + final IntsRef target = Util.toUTF32(fragment, new IntsRef()); + if (target.length == 0) { + throw new IllegalArgumentException("fragment must have at least one character; got \"" + fragment + "\""); + } + + // We assume this below: + assert target.offset == 0; + + int pathOrd = 0; + + // Root path: + queue.add(new Path(pathOrd++, null, 0, fst.getFirstArc(new FST.Arc()), 0, false, -1)); + + final List topN = new ArrayList(); + + final FST.BytesReader fstReader = fst.getBytesReader(0); + + final Long NO_OUTPUT = fst.outputs.getNoOutput(); + + int leastCost = -1; + int numPop = 0; + + final FST.Arc scratchArc = new FST.Arc(); + + while (true) { + final Path path = queue.pop(); + if (path == null) { + break; + } + numPop++; + if (VERBOSE) { + System.out.println("\n[" + queue.queue.size() + "] path " + path); + } + + if (path.inputNode == target.length) { + + // We've exhausted the input: + if (path.fstNode.label == FST.END_LABEL) { + + // We've exhausted the FST too, so this is a match: + if (topN.size() == 0) { + leastCost = path.cost; + } else { + assert path.cost >= leastCost || exactFirst; + assert path.cost >= topN.get(topN.size()-1).cost || (exactFirst&& topN.size() == 1); + } + if (VERBOSE) { + System.out.println(" final: " + fragmentToString(path.getFragment()) + " " + (path.cost-leastCost) + ";" + (path.hasEdit ? " [fuzzy]" : "")); + } + topN.add(path); + if (topN.size() == count) { + // Done! + //System.out.println("Q: " + queue.queue.size()); + break; + } + + continue; + } + + // Add all tail extensions from here: + fst.readFirstTargetArc(path.fstNode, scratchArc, fstReader); + //System.out.println(" tail"); + + Path path2 = path; + + // Since we are in the wFST tail (the .* part of the + // fragment) we know this path must complete via + // 0-cost arcs to a final node: + while(true) { + Path bestPath = null; + while (true) { + final int arcCost = scratchArc.output.intValue(); + if (VERBOSE) { + System.out.println(" complete fstLabel=" + labelToChar(scratchArc.label) + " arcCost=" + arcCost); + } + final Path nextPath = new Path(pathOrd++, + path2, + path2.inputNode, + new FST.Arc().copyFrom(scratchArc), + path2.cost + arcCost, + path2.hasEdit, + -1); + if (scratchArc.output == NO_OUTPUT && bestPath == null) { + // Skip the queue (we will pursue it immediately): + bestPath = nextPath; + if (VERBOSE) { + System.out.println(" -> bestPath " + bestPath); + } + } else { + //System.out.println(" ac=" + arcCost); + // Add to queue (we may pursue it later): + queue.add(nextPath); + } + if (scratchArc.isLast()) { + break; + } + fst.readNextArc(scratchArc, fstReader); + } + assert bestPath != null; + + if (bestPath.fstNode.label == FST.END_LABEL) { + if (topN.size() == 0) { + leastCost = bestPath.cost; + } else { + assert bestPath.cost >= leastCost || exactFirst; + assert bestPath.cost >= topN.get(topN.size()-1).cost || (exactFirst&& topN.size() == 1); + } + if (VERBOSE) { + System.out.println(" final2: " + fragmentToString(bestPath.getFragment()) + " " + (path.cost-leastCost) + ";" + (bestPath.hasEdit ? " [fuzzy]" : "")); + } + topN.add(bestPath); + break; + } else { + if (bestPath.inputNode == target.length) { + final IntsRef prefix = bestPath.getFragment(); + if (queue.prefixSeen.contains(prefix)) { + if (VERBOSE) { + System.out.println(" skip prefix " + fragmentToString(prefix) + ": already seen"); + } + break; + } else { + if (VERBOSE) { + System.out.println(" add prefix " + fragmentToString(prefix)); + } + queue.prefixSeen.add(prefix); + } + } + path2 = bestPath; + if (VERBOSE) { + System.out.println("\n[" + queue.queue.size() + "] bestPath " + path2); + } + fst.readFirstTargetArc(path2.fstNode, scratchArc, fstReader); + } + } + + if (topN.size() == count) { + // Done! + //System.out.println("Q: " + queue.queue.size()); + break; + } + + } else { + assert path.inputNode < target.length; + + // Still "inside" the target: + + if (path.hasEdit || path.inputNode == 0) { + // Path already has an edit, or we are at first + // label (where we disallow an edit); just extend + // by an exact match in the FST: + //System.out.println(" lb=" + (char) target.ints[path.inputNode] + " pos=" + path.inputNode + " frag=" + Util.toBytesRef(target, new BytesRef()).utf8ToString()); + final FST.Arc nextFSTNode = fst.findTargetArc(target.ints[path.inputNode], + path.fstNode, + new FST.Arc(), + fstReader); + if (nextFSTNode != null) { + final int arcCost = nextFSTNode.output.intValue(); + if (VERBOSE) { + System.out.println(" exact label=" + labelToChar(nextFSTNode.label) + " arcCost=" + arcCost); + } + queue.add(new Path(pathOrd++, + path, + path.inputNode+1, + nextFSTNode, + path.cost + arcCost, + path.hasEdit, + -1)); + + if (path.inputNode < target.length-2 && + path.priorDeleteLabel != -1 && + path.priorDeleteLabel == target.ints[1+path.inputNode] && + path.priorDeleteLabel != target.ints[path.inputNode]) { + + assert path.hasEdit; + if (VERBOSE) { + System.out.println(" TRANS inputLabel=" + (char) target.ints[1+path.inputNode]); + } + + // Transposition + queue.add(new Path(pathOrd++, + path, + path.inputNode+2, + nextFSTNode, + path.cost + arcCost, + true, + -1)); + } + + } else { + if (VERBOSE) { + System.out.println(" no exact label=" + (char) target.ints[path.inputNode]); + } + } + } else { + // Allow one edit: + boolean foundExact = false; + + fst.readFirstTargetArc(path.fstNode, scratchArc, fstReader); + while (true) { + + // nocommit the label can be END_LABEL here... + + // Deletion (don't advance input but do advance + // FST): + final int arcCost = scratchArc.output.intValue(); + // nocommit + if (scratchArc.label != FST.END_LABEL && + (true || path.inputNode == target.length-1 || + target.ints[path.inputNode] != target.ints[path.inputNode+1])) { + if (VERBOSE) { + System.out.println(" DEL fstLabel=" + labelToChar(scratchArc.label)); + } + queue.add(new Path(pathOrd++, + path, + path.inputNode, + new FST.Arc().copyFrom(scratchArc), + path.cost + EDIT_PENALTY + arcCost, + true, + scratchArc.label)); + } else { + if (VERBOSE) { + System.out.println(" SKIP POINTLESS DEL=" + labelToChar(scratchArc.label)); + } + } + + // Substitution (advance input and FST), only if + // the label doesn't match): + if (scratchArc.label != target.ints[path.inputNode]) { + if (scratchArc.label != FST.END_LABEL) { + if (VERBOSE) { + System.out.println(" SUB inputLabel=" + (char) target.ints[path.inputNode] + " fstLabel=" + labelToChar(scratchArc.label)); + } + queue.add(new Path(pathOrd++, + path, + path.inputNode+1, + new FST.Arc().copyFrom(scratchArc), + path.cost + EDIT_PENALTY + arcCost, + true, + -1)); + } + } else { + // Exact match: + if (VERBOSE) { + System.out.println(" exact label=" + (char) target.ints[path.inputNode]); + } + queue.add(new Path(pathOrd++, + path, + path.inputNode+1, + new FST.Arc().copyFrom(scratchArc), + path.cost + arcCost, + path.hasEdit, + -1)); + foundExact = true; + } + + if (scratchArc.isLast()) { + break; + } + fst.readNextArc(scratchArc, fstReader); + } + + final boolean allowIns; + if (path.inputNode == target.length-1) { + allowIns = true; + } else { + allowIns = target.ints[path.inputNode] != target.ints[path.inputNode+1]; + } + + if (allowIns) { + // Insertion (do advance input but don't advance + // FST): + if (VERBOSE) { + System.out.println(" INS inputLabel=" + (char) target.ints[path.inputNode]); + } + queue.add(new Path(pathOrd++, + path, + path.inputNode+1, + path.fstNode, + path.cost + EDIT_PENALTY, + true, + -1)); + } else { + if (VERBOSE) { + System.out.println("SKIP POINTLESS INS"); + } + } + } + } + } + + if (VERBOSE) { + System.out.println(" " + fragment + ": " + queue.numPush + " pushes; " + numPop + " pops"); + } + + final List result = new ArrayList(topN.size()); + int lastCost = -1; + final int limit = topN.size(); + // for(final Path path : topN) { + for(int idx=0;idx= lastCost || exactFirst; + lastCost = path.cost; + final IntsRef answer = path.getFragment(); + assert answer.length > 0; + assert answer.ints[answer.offset+answer.length-1] == FST.END_LABEL; + answer.length--; + result.add(new Completion(answer, path.cost, path.hasEdit)); + //if (queue.numPush > 300 || VERBOSE) { + //System.out.println(" " + result.get(result.size()-1)); + //} + } + + return result; + } + + @Override + public List lookup(CharSequence key, boolean higherWeightsFirst, int num) { + if (!higherWeightsFirst) { + throw new IllegalArgumentException("higherWeightsFirst must be true"); + } + final List completions; + try { + completions = lookup(key, num); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } + final ArrayList results = new ArrayList(completions.size()); + for (Completion c : completions) { + results.add(new LookupResult(UnicodeUtil.newString(c.term.ints, c.term.offset, c.term.length), + c.cost)); + } + return results; + } + + @Override + public boolean store(OutputStream output) throws IOException { + throw new UnsupportedOperationException(""); + } + + @Override + public boolean load(InputStream input) throws IOException { + throw new UnsupportedOperationException(""); + } + + private final class FuzzyTermFreqIteratorWrapper extends SortedTermFreqIteratorWrapper { + + FuzzyTermFreqIteratorWrapper(TermFreqIterator source, + Comparator comparator) throws IOException { + super(source, comparator, true); + } + + @Override + protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { + if (spare.length + 4 >= buffer.length) { + buffer = ArrayUtil.grow(buffer, spare.length + 4); + } + output.reset(buffer); + output.writeBytes(spare.bytes, spare.offset, spare.length); + output.writeInt((int)weight); + writer.write(buffer, 0, output.getPosition()); + } + + @Override + protected long decode(BytesRef scratch, ByteArrayDataInput tmpInput) { + tmpInput.reset(scratch.bytes); + scratch.length -= 4; + tmpInput.skipBytes(scratch.length); + return tmpInput.readInt(); + } + } +} Index: modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java =================================================================== --- modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (revision 1298343) +++ modules/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (working copy) @@ -32,6 +32,7 @@ import org.apache.lucene.util.*; import org.apache.lucene.search.suggest.Lookup; import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; +import org.apache.lucene.search.suggest.fuzzyfst.FuzzyFSTCompletionLookup; import org.apache.lucene.search.suggest.fst.WFSTCompletionLookup; import org.apache.lucene.search.suggest.jaspell.JaspellLookup; import org.apache.lucene.search.suggest.tst.TSTLookup; @@ -42,17 +43,22 @@ /** * Benchmarks tests for implementations of {@link Lookup} interface. */ -@Ignore("COMMENT ME TO RUN BENCHMARKS!") +// nocommit +//@Ignore("COMMENT ME TO RUN BENCHMARKS!") public class LookupBenchmarkTest extends LuceneTestCase { @SuppressWarnings("unchecked") private final List> benchmarkClasses = Arrays.asList( + FuzzyFSTCompletionLookup.class, JaspellLookup.class, TSTLookup.class, FSTCompletionLookup.class, WFSTCompletionLookup.class); + // nocommit private final static int rounds = 15; private final static int warmup = 5; + //private final static int rounds = 1; + //private final static int warmup = 0; private final int num = 7; private final boolean onlyMorePopular = true; @@ -123,6 +129,7 @@ cls.getSimpleName(), dictionaryInput.length, result.average.toString())); + //break; } } @@ -138,6 +145,7 @@ String.format(Locale.ENGLISH, "%-15s size[B]:%,13d", lookup.getClass().getSimpleName(), rue.estimateRamUsage(lookup))); + //break; } } @@ -201,6 +209,8 @@ int v = 0; for (String term : input) { v += lookup.lookup(term, onlyMorePopular, num).size(); + //v += lookup.lookup("popo", true, 20).size(); + //break; } return v; } @@ -212,6 +222,7 @@ input.size(), result.average.toString(), input.size() / result.average.avg)); + break; } } Index: modules/suggest/src/test/org/apache/lucene/search/suggest/fuzzyfst/TestFuzzyFSTCompletionLookup.java =================================================================== --- modules/suggest/src/test/org/apache/lucene/search/suggest/fuzzyfst/TestFuzzyFSTCompletionLookup.java (revision 0) +++ modules/suggest/src/test/org/apache/lucene/search/suggest/fuzzyfst/TestFuzzyFSTCompletionLookup.java (working copy) @@ -0,0 +1,380 @@ +package org.apache.lucene.search.suggest.fuzzyfst; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.lucene.search.suggest.TermFreq; +import org.apache.lucene.search.suggest.TermFreqArrayIterator; +import org.apache.lucene.search.suggest.fuzzyfst.FuzzyFSTCompletionLookup.Completion; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntsRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.fst.Util; + +public class TestFuzzyFSTCompletionLookup extends LuceneTestCase { + + private FuzzyFSTCompletionLookup suggest; + + private void build(TermFreq... answers) throws IOException { + suggest = new FuzzyFSTCompletionLookup(true); + suggest.build(new TermFreqArrayIterator(answers)); + /* + Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"), "UTF-8"); + Util.toDot(lookup.getFST(), w, false, false); + w.close(); + */ + } + + private void verify(String fragment, + String... expected) throws IOException { + List results = suggest.lookup(fragment, 2*expected.length); + System.out.println("GOT: " + results); + assertEquals(expected.length, results.size()); + for(int idx=0;idx answers = new ArrayList(); + final Set seen = new HashSet(); + for(int i=0;i() { + @Override + public int compare(TermFreq a, TermFreq b) { + return a.term.compareTo(b.term); + } + }); + if (VERBOSE) { + System.out.println("\nTEST: targets"); + for(TermFreq tf : answers) { + System.out.println(" " + tf.term.utf8ToString() + " freq=" + tf.v); + } + } + suggest = new FuzzyFSTCompletionLookup(true); + Collections.shuffle(answers, random); + suggest.build(new TermFreqArrayIterator(answers.toArray(new TermFreq[answers.size()]))); + + final int ITERS = atLeast(100); + for(int iter=0;iter actual = suggest.lookup(frag, NUM); + if (VERBOSE) { + System.out.println(" actual: " + actual.size()); + for(Completion c : actual) { + System.out.println(" " + c); + } + } + + // nocommit must fix lookup to tie break properly!!: + Collections.sort(actual, new CompareByCostThenAlpha()); + + final int limit = Math.min(expected.size(), actual.size()); + for(int ans=0;ans slowFuzzyMatch(List answers, String frag) { + final List results = new ArrayList(); + final int fragLength = frag.length(); + for(TermFreq tf : answers) { + if (tf.term.bytes[0] == (byte) frag.charAt(0)) { + // OK: first char matches + final int len = tf.term.length; + if (len >= fragLength-1) { + // OK it's possible: + int d; + final String s = tf.term.utf8ToString(); + if (fragLength == 1) { + d = 0; + } else if (len == fragLength-1) { + d = getDistance(frag, s); + } else { + //System.out.println(" check s=" + s); + d = getDistance(frag, s.substring(0, fragLength)); + //System.out.println(" d=" + d + " s=" + s.substring(0, fragLength)); + if (d > 1) { + d = getDistance(frag, s.substring(0, fragLength-1)); + //System.out.println(" d=" + d + " s=" + s.substring(0, fragLength-1)); + if (d > 1 && s.length() >= fragLength+1) { + d = getDistance(frag, s.substring(0, fragLength+1)); + //System.out.println(" d=" + d + " s=" + s.substring(0, fragLength+1)); + } + } + } + + if (d == 0) { + results.add(new Completion(Util.toIntsRef(tf.term, new IntsRef()), + (int) (1000000000 - tf.v), + false)); + } else if (d == 1) { + results.add(new Completion(Util.toIntsRef(tf.term, new IntsRef()), + (int) (1000000000 - tf.v + FuzzyFSTCompletionLookup.EDIT_PENALTY), + true)); + } + } + } + + Collections.sort(results, new CompareByCostThenAlpha()); + } + + return results; + } + + public class CompareByCostThenAlpha implements Comparator { + @Override + public int compare(Completion a, Completion b) { + if (a.cost < b.cost) { + return -1; + } else if (a.cost > b.cost) { + return 1; + } else { + final int c = a.term.compareTo(b.term); + assert c != 0: "term=" + Util.toBytesRef(a.term, new BytesRef()).utf8ToString(); + return c; + } + } + } + + // NOTE: copied from + // modules/suggest/src/java/org/apache/lucene/search/spell/LuceneLevenshteinDistance.java + // and tweaked to return the edit distance not the float + // lucene measure + + /* Finds unicode (code point) Levenstein (edit) distance + * between two strings, including transpositions. */ + public int getDistance(String target, String other) { + IntsRef targetPoints; + IntsRef otherPoints; + int n; + int d[][]; // cost array + + // NOTE: if we cared, we could 3*m space instead of m*n space, similar to + // what LevenshteinDistance does, except cycling thru a ring of three + // horizontal cost arrays... but this comparator is never actually used by + // DirectSpellChecker, its only used for merging results from multiple shards + // in "distributed spellcheck", and its inefficient in other ways too... + + // cheaper to do this up front once + targetPoints = toIntsRef(target); + otherPoints = toIntsRef(other); + n = targetPoints.length; + final int m = otherPoints.length; + d = new int[n+1][m+1]; + + if (n == 0 || m == 0) { + if (n == m) { + return 0; + } + else { + return Math.max(n, m); + } + } + + // indexes into strings s and t + int i; // iterates through s + int j; // iterates through t + + int t_j; // jth character of t + + int cost; // cost + + for (i = 0; i<=n; i++) { + d[i][0] = i; + } + + for (j = 0; j<=m; j++) { + d[0][j] = j; + } + + for (j = 1; j<=m; j++) { + t_j = otherPoints.ints[j-1]; + + for (i=1; i<=n; i++) { + cost = targetPoints.ints[i-1]==t_j ? 0 : 1; + // minimum of cell to the left+1, to the top+1, diagonally left and up +cost + d[i][j] = Math.min(Math.min(d[i-1][j]+1, d[i][j-1]+1), d[i-1][j-1]+cost); + // transposition + if (i > 1 && j > 1 && targetPoints.ints[i-1] == otherPoints.ints[j-2] && targetPoints.ints[i-2] == otherPoints.ints[j-1]) { + d[i][j] = Math.min(d[i][j], d[i-2][j-2] + cost); + } + } + } + + return d[n][m]; + } + + private static IntsRef toIntsRef(String s) { + IntsRef ref = new IntsRef(s.length()); // worst case + int utf16Len = s.length(); + for (int i = 0, cp = 0; i < utf16Len; i += Character.charCount(cp)) { + cp = ref.ints[ref.length++] = Character.codePointAt(s, i); + } + return ref; + } +}