diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java index 8925cca..d617385 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/BytesRefList.java @@ -18,23 +18,34 @@ package org.apache.lucene.search.suggest; */ import java.io.IOException; +import java.util.Arrays; import java.util.Comparator; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import org.apache.lucene.util.Counter; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.SorterTemplate; -final class BytesRefList { +/** + * + * @lucene.internal + * @lucene.experimental + */ +public final class BytesRefList { private final ByteBlockPool pool; private int[] offsets = new int[1]; private int currentElement = 0; private int currentOffset = 0; + private final Counter bytesUsed = Counter.newCounter(false); public BytesRefList() { - this(new ByteBlockPool(new ByteBlockPool.DirectAllocator())); + this.pool = new ByteBlockPool(new ByteBlockPool.DirectTrackingAllocator(bytesUsed)); + pool.nextBuffer(); + bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + RamUsageEstimator.NUM_BYTES_INT); } public BytesRefList(ByteBlockPool pool) { @@ -42,9 +53,18 @@ final class BytesRefList { pool.nextBuffer(); } + public void clear() { + currentElement = 0; + currentOffset = 0; + Arrays.fill(offsets, 0); + pool.reset(); + } + public int append(BytesRef bytes) { if (currentElement >= offsets.length) { + int oldLen = offsets.length; offsets = ArrayUtil.grow(offsets, offsets.length + 1); + bytesUsed.addAndGet((offsets.length - oldLen)*RamUsageEstimator.NUM_BYTES_INT); } pool.copy(bytes); offsets[currentElement++] = currentOffset; @@ -91,7 +111,9 @@ final class BytesRefList { } }; } - + public long bytesUsed() { + return bytesUsed.get(); + } public int[] sort(final Comparator comp) { final int[] orderdEntries = new int[size()]; for (int i = 0; i < orderdEntries.length; i++) { diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java index 059e1c2..49f9f76 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java @@ -75,7 +75,11 @@ public class FileDictionary implements Dictionary { String[] fields = line.split("\t"); if (fields.length > 1) { // keep reading floats for bw compat - curFreq = (int)Float.parseFloat(fields[1]); + try { + curFreq = Long.parseLong(fields[1]); + } catch (NumberFormatException e) { + curFreq = (long)Double.parseDouble(fields[1]); + } spare.copyChars(fields[0]); } else { spare.copyChars(line); diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java index eff3ee7..dc39ec9 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java @@ -35,9 +35,9 @@ public abstract class Lookup { */ public static final class LookupResult implements Comparable { public final CharSequence key; - public final float value; + public final long value; - public LookupResult(CharSequence key, float value) { + public LookupResult(CharSequence key, long value) { this.key = key; this.value = value; } @@ -124,23 +124,8 @@ public abstract class Lookup { */ public abstract List lookup(CharSequence key, boolean onlyMorePopular, int num); - /** - * Modify the lookup data by recording additional data. Optional operation. - * @param key new lookup key - * @param value value to associate with this key - * @return true if new key is added, false if it already exists or operation - * is not supported. - */ - public abstract boolean add(CharSequence key, Object value); /** - * Get value associated with a specific key. - * @param key lookup key - * @return associated value - */ - public abstract Object get(CharSequence key); - - /** * Persist the constructed lookup data to a directory. Optional operation. * @param output {@link OutputStream} to write the data to. * @return true if successful, false if unsuccessful or not supported. @@ -173,4 +158,18 @@ public abstract class Lookup { * @throws IOException when fatal IO error occurs. */ public abstract boolean load(File storeDir) throws IOException; + + + protected Number weightAsNumber(long weight) { + if (weight < 0) { + if (weight > Integer.MIN_VALUE) { + return weight > Short.MIN_VALUE ? Short.valueOf((short)weight) : Integer.valueOf((int)weight); + } + } else { + if (weight < Integer.MAX_VALUE) { + return weight < Short.MAX_VALUE ? Short.valueOf((short)weight) : Integer.valueOf((int)weight); + } + } + return Long.valueOf(weight); + } } diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java index 2380724..2483223 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/SortedTermFreqIteratorWrapper.java @@ -17,45 +17,165 @@ package org.apache.lucene.search.suggest; * limitations under the License. */ +import java.io.File; import java.io.IOException; import java.util.Comparator; import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.search.suggest.fst.Sort; +import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesReader; +import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; /** - * This wrapper buffers incoming elements and makes sure they are sorted in - * ascending lexicographic order. + * This wrapper buffers incoming elements and makes sure they are sorted based on given comparator. */ -public class SortedTermFreqIteratorWrapper extends BufferingTermFreqIteratorWrapper { - // TODO keep this for now - but the consumer should really sort this stuff on disk with sorter... - private final int[] sortedOrds; - private int currentOrd = -1; - private final BytesRef spare = new BytesRef(); - private final Comparator comp; - - public SortedTermFreqIteratorWrapper(TermFreqIterator source, Comparator comp) throws IOException { - super(source); - this.sortedOrds = entries.sort(comp); - this.comp = comp; +public class SortedTermFreqIteratorWrapper implements TermFreqIterator { + + private final TermFreqIterator source; + private File tempInput; + private File tempSorted; + private final ByteSequencesReader reader; + private boolean done = false; + + private long weight; + private final BytesRef scratch = new BytesRef(); + private final Comparator comparator; + + public SortedTermFreqIteratorWrapper(TermFreqIterator source, Comparator comparator) throws IOException { + this(source, comparator, false); } - - @Override - public long weight() { - return freqs[currentOrd]; + + public SortedTermFreqIteratorWrapper(TermFreqIterator source, Comparator comparator, boolean compareRawBytes) throws IOException { + this.source = source; + this.comparator = comparator; + this.reader = sort(compareRawBytes ? comparator : new BytesOnlyComparator(this.comparator)); } - + @Override public BytesRef next() throws IOException { - if (++curPos < entries.size()) { - return entries.get(spare, (currentOrd = sortedOrds[curPos])); + boolean success = false; + if (done) { + return null; + } + try { + ByteArrayDataInput input = new ByteArrayDataInput(); + if (reader.read(scratch)) { + weight = decode(scratch, input); + success = true; + return scratch; + } + close(); + success = done = true; + return null; + } finally { + if (!success) { + done = true; + close(); + } } - return null; } - + @Override public Comparator getComparator() { - return comp; + return comparator; + } + + @Override + public long weight() { + return weight; + } + + private Sort.ByteSequencesReader sort(Comparator comparator) throws IOException { + String prefix = getClass().getSimpleName(); + File directory = Sort.defaultTempDir(); + tempInput = File.createTempFile(prefix, ".input", directory); + tempSorted = File.createTempFile(prefix, ".sorted", directory); + + final Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput); + boolean success = false; + try { + BytesRef spare; + byte[] buffer = new byte[0]; + ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); + + while ((spare = source.next()) != null) { + encode(writer, output, buffer, spare, source.weight()); + } + writer.close(); + new Sort(comparator).sort(tempInput, tempSorted); + ByteSequencesReader reader = new Sort.ByteSequencesReader(tempSorted); + success = true; + return reader; + + } finally { + if (success) { + IOUtils.close(writer); + } else { + try { + IOUtils.closeWhileHandlingException(writer); + } finally { + close(); + } + } + + } + } + + private void close() throws IOException { + if (tempInput != null) { + tempInput.delete(); + } + if (tempSorted != null) { + tempSorted.delete(); + } + IOUtils.close(reader); + } + + private final static class BytesOnlyComparator implements Comparator { + + final Comparator other; + private final BytesRef leftScratch = new BytesRef(); + private final BytesRef rightScratch = new BytesRef(); + + public BytesOnlyComparator(Comparator other) { + this.other = other; + } + + @Override + public int compare(BytesRef left, BytesRef right) { + wrap(leftScratch, left); + wrap(rightScratch, right); + return other.compare(leftScratch, rightScratch); + } + + private void wrap(BytesRef wrapper, BytesRef source) { + wrapper.bytes = source.bytes; + wrapper.offset = source.offset; + wrapper.length = source.length - 8; + + } + } + + protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { + if (spare.length + 8 >= buffer.length) { + buffer = ArrayUtil.grow(buffer, spare.length + 8); + } + output.reset(buffer); + output.writeBytes(spare.bytes, spare.offset, spare.length); + output.writeLong(weight); + writer.write(buffer, 0, output.getPosition()); + } + + protected long decode(BytesRef scratch, ByteArrayDataInput tmpInput) { + tmpInput.reset(scratch.bytes); + tmpInput.skipBytes(scratch.length - 8); // suggestion + separator + scratch.length -= 8; // sep + long + return tmpInput.readLong(); } } diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java index 59fdc4c..b4d5bf3 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java @@ -159,10 +159,10 @@ public class FSTCompletion { * @param utf8 * The sequence of utf8 bytes to follow. * - * @return Returns the bucket number of the match or null if no + * @return Returns the bucket number of the match or -1 if no * match was found. */ - private Integer getExactMatchStartingFromRootArc( + private int getExactMatchStartingFromRootArc( int rootArcIndex, BytesRef utf8) { // Get the UTF-8 bytes representation of the input key. try { @@ -186,7 +186,7 @@ public class FSTCompletion { } // No match. - return null; + return -1; } /** @@ -273,8 +273,8 @@ public class FSTCompletion { // exact match, if requested. if (exactFirst) { if (!checkExistingAndReorder(res, key)) { - Integer exactMatchBucket = getExactMatchStartingFromRootArc(i, key); - if (exactMatchBucket != null) { + int exactMatchBucket = getExactMatchStartingFromRootArc(i, key); + if (exactMatchBucket != -1) { // Insert as the first result and truncate at num. while (res.size() >= num) { res.remove(res.size() - 1); @@ -385,10 +385,10 @@ public class FSTCompletion { } /** - * Returns the bucket assigned to a given key (if found) or null if + * Returns the bucket assigned to a given key (if found) or -1 if * no exact match exists. */ - public Integer getBucket(CharSequence key) { + public int getBucket(CharSequence key) { return getExactMatchStartingFromRootArc(0, new BytesRef(key)); } diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java index aee2ea1..7808be6 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java @@ -171,7 +171,7 @@ public class FSTCompletionLookup extends Lookup { } output.reset(buffer); - output.writeInt(FloatMagic.toSortable(tfit.weight())); + output.writeInt(encodeWeight(tfit.weight())); output.writeBytes(spare.bytes, spare.offset, spare.length); writer.write(buffer, 0, output.getPosition()); } @@ -188,13 +188,13 @@ public class FSTCompletionLookup extends Lookup { reader = new Sort.ByteSequencesReader(tempSorted); long line = 0; int previousBucket = 0; - float previousScore = 0; + int previousScore = 0; ByteArrayDataInput input = new ByteArrayDataInput(); BytesRef tmp1 = new BytesRef(); BytesRef tmp2 = new BytesRef(); while (reader.read(tmp1)) { input.reset(tmp1.bytes); - float currentScore = FloatMagic.fromSortable(input.readInt()); + int currentScore = input.readInt(); int bucket; if (line > 0 && currentScore == previousScore) { @@ -230,6 +230,14 @@ public class FSTCompletionLookup extends Lookup { tempSorted.delete(); } } + + /** weight -> cost */ + private static int encodeWeight(long value) { + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + throw new UnsupportedOperationException("cannot encode value: " + value); + } + return (int)value; + } @Override public List lookup(CharSequence key, boolean higherWeightsFirst, int num) { @@ -250,19 +258,9 @@ public class FSTCompletionLookup extends Lookup { return results; } - @Override - public boolean add(CharSequence key, Object value) { - // Not supported. - return false; - } - - @Override public Object get(CharSequence key) { - Integer bucket = normalCompletion.getBucket(key); - if (bucket == null) - return null; - else - return (float) normalCompletion.getBucket(key) / normalCompletion.getBucketCount(); + final int bucket = normalCompletion.getBucket(key); + return bucket == -1 ? null : Long.valueOf(bucket); } /** diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FloatMagic.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FloatMagic.java deleted file mode 100644 index 1658356..0000000 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FloatMagic.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.apache.lucene.search.suggest.fst; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.util.NumericUtils; - -/** - * Converts normalized float representations ({@link Float#floatToIntBits(float)}) - * into integers that are directly sortable in int4 representation (or unsigned values or - * after promoting to a long with higher 32-bits zeroed). - */ -class FloatMagic { - /** - * Convert a float to a directly sortable unsigned integer. For sortable signed - * integers, see {@link NumericUtils#floatToSortableInt(float)}. - */ - public static int toSortable(float f) { - return floatBitsToUnsignedOrdered(Float.floatToRawIntBits(f)); - } - - /** - * Back from {@link #toSortable(float)} to float. - */ - public static float fromSortable(int v) { - return Float.intBitsToFloat(unsignedOrderedToFloatBits(v)); - } - - /** - * Convert float bits to directly sortable bits. - * Normalizes all NaNs to canonical form. - */ - static int floatBitsToUnsignedOrdered(int v) { - // Canonicalize NaN ranges. I assume this check will be faster here than - // (v == v) == false on the FPU? We don't distinguish between different - // flavors of NaNs here (see http://en.wikipedia.org/wiki/NaN). I guess - // in Java this doesn't matter much anyway. - if ((v & 0x7fffffff) > 0x7f800000) { - // Apply the logic below to a canonical "quiet NaN" - return 0x7fc00000 ^ 0x80000000; - } - - if (v < 0) { - // Reverse the order of negative values and push them before positive values. - return ~v; - } else { - // Shift positive values after negative, but before NaNs, they're sorted already. - return v ^ 0x80000000; - } - } - - /** - * Back from {@link #floatBitsToUnsignedOrdered(int)}. - */ - static int unsignedOrderedToFloatBits(int v) { - if (v < 0) - return v & ~0x80000000; - else - return ~v; - } -} diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java index 47942ed..ec73c65 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/Sort.java @@ -20,15 +20,10 @@ package org.apache.lucene.search.suggest.fst; import java.io.*; import java.util.*; +import org.apache.lucene.search.suggest.BytesRefList; import org.apache.lucene.util.*; import org.apache.lucene.util.PriorityQueue; -// TODO: the buffer is currently byte[][] which with very small arrays will terribly overallocate -// memory (alignments) and make GC very happy. -// -// We could move it to a single byte[] + and use custom sorting, but we'd need to check if this -// yields any improvement first. - /** * On-disk sorting of byte arrays. Each byte array (entry) is a composed of the following * fields: @@ -59,11 +54,6 @@ public final class Sort { */ public final static int MAX_TEMPFILES = 128; - /** - * Minimum slot buffer expansion. - */ - private final static int MIN_EXPECTED_GROWTH = 1000; - /** * A bit more descriptive unit for constructors. * @@ -112,21 +102,6 @@ public final class Sort { } /** - * byte[] in unsigned byte order. - */ - static final Comparator unsignedByteOrderComparator = new Comparator() { - public int compare(byte[] left, byte[] right) { - final int max = Math.min(left.length, right.length); - for (int i = 0, j = 0; i < max; i++, j++) { - int diff = (left[i] & 0xff) - (right[j] & 0xff); - if (diff != 0) - return diff; - } - return left.length - right.length; - } - }; - - /** * Sort info (debugging mostly). */ public class SortInfo { @@ -149,14 +124,15 @@ public final class Sort { } } - private final static byte [][] EMPTY = new byte [0][]; - private final BufferSize ramBufferSize; private final File tempDirectory; - - private byte [][] buffer = new byte [0][]; + + private final BytesRefList buffer = new BytesRefList(); private SortInfo sortInfo; private int maxTempFiles; + private final Comparator comparator; + + public static final Comparator DEFAULT_COMPARATOR = BytesRef.getUTF8SortedAsUnicodeComparator(); /** * Defaults constructor. @@ -165,13 +141,17 @@ public final class Sort { * @see BufferSize#automatic() */ public Sort() throws IOException { - this(BufferSize.automatic(), defaultTempDir(), MAX_TEMPFILES); + this(DEFAULT_COMPARATOR, BufferSize.automatic(), defaultTempDir(), MAX_TEMPFILES); + } + + public Sort(Comparator comparator) throws IOException { + this(comparator, BufferSize.automatic(), defaultTempDir(), MAX_TEMPFILES); } /** * All-details constructor. */ - public Sort(BufferSize ramBufferSize, File tempDirectory, int maxTempfiles) { + public Sort(Comparator comparator, BufferSize ramBufferSize, File tempDirectory, int maxTempfiles) { if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) { throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + ramBufferSize.bytes); } @@ -183,6 +163,7 @@ public final class Sort { this.ramBufferSize = ramBufferSize; this.tempDirectory = tempDirectory; this.maxTempFiles = maxTempfiles; + this.comparator = comparator; } /** @@ -283,23 +264,26 @@ public final class Sort { /** Sort a single partition in-memory. */ protected File sortPartition(int len) throws IOException { - byte [][] data = this.buffer; + BytesRefList data = this.buffer; File tempFile = File.createTempFile("sort", "partition", tempDirectory); long start = System.currentTimeMillis(); - Arrays.sort(data, 0, len, unsignedByteOrderComparator); + int[] sort = buffer.sort(comparator); sortInfo.sortTime += (System.currentTimeMillis() - start); - ByteSequencesWriter out = new ByteSequencesWriter(tempFile); + final ByteSequencesWriter out = new ByteSequencesWriter(tempFile); + final BytesRef spare = new BytesRef(); + assert len == sort.length : "expected: " + len + " but was: " + sort.length; try { - for (int i = 0; i < len; i++) { - assert data[i].length <= Short.MAX_VALUE; - out.write(data[i]); + for (int i = 0; i < sort.length; i++) { + data.get(spare, sort[i]); + assert spare.length <= Short.MAX_VALUE; + out.write(spare); } out.close(); // Clean up the buffer for the next partition. - this.buffer = EMPTY; + data.clear(); return tempFile; } finally { IOUtils.close(out); @@ -314,7 +298,7 @@ public final class Sort { PriorityQueue queue = new PriorityQueue(merges.size()) { protected boolean lessThan(FileAndTop a, FileAndTop b) { - return a.current.compareTo(b.current) < 0; + return comparator.compare(a.current, b.current) < 0; } }; @@ -359,33 +343,18 @@ public final class Sort { /** Read in a single partition of data */ int readPartition(ByteSequencesReader reader) throws IOException { long start = System.currentTimeMillis(); - - // We will be reallocating from scratch. - Arrays.fill(this.buffer, null); - - int bytesLimit = this.ramBufferSize.bytes; - byte [][] data = this.buffer; - byte[] line; - int linesRead = 0; - while ((line = reader.read()) != null) { - if (linesRead + 1 >= data.length) { - data = Arrays.copyOf(data, - ArrayUtil.oversize(linesRead + MIN_EXPECTED_GROWTH, - RamUsageEstimator.NUM_BYTES_OBJECT_REF)); - } - data[linesRead++] = line; - + final BytesRef scratch = new BytesRef(); + while ((scratch.bytes = reader.read()) != null) { + scratch.length = scratch.bytes.length; + buffer.append(scratch); // Account for the created objects. // (buffer slots do not account to buffer size.) - bytesLimit -= line.length + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; - if (bytesLimit < 0) { + if (ramBufferSize.bytes < buffer.bytesUsed()) { break; } } - this.buffer = data; - sortInfo.readTime += (System.currentTimeMillis() - start); - return linesRead; + return buffer.size(); } static class FileAndTop { diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java index d286e48..b97cc89 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java @@ -23,10 +23,13 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.List; import org.apache.lucene.search.spell.TermFreqIterator; import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.SortedTermFreqIteratorWrapper; +import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ByteArrayDataOutput; import org.apache.lucene.store.InputStreamDataInput; @@ -101,72 +104,27 @@ public class WFSTCompletionLookup extends Lookup { @Override public void build(TermFreqIterator iterator) throws IOException { - String prefix = getClass().getSimpleName(); - File directory = Sort.defaultTempDir(); - File tempInput = File.createTempFile(prefix, ".input", directory); - File tempSorted = File.createTempFile(prefix, ".sorted", directory); - - Sort.ByteSequencesWriter writer = new Sort.ByteSequencesWriter(tempInput); - Sort.ByteSequencesReader reader = null; BytesRef scratch = new BytesRef(); - - boolean success = false; - try { - byte [] buffer = new byte [0]; - ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); - BytesRef spare; - while ((spare = iterator.next()) != null) { - if (spare.length + 5 >= buffer.length) { - buffer = ArrayUtil.grow(buffer, spare.length + 5); - } - - output.reset(buffer); - output.writeBytes(spare.bytes, spare.offset, spare.length); - output.writeByte((byte)0); // separator: not used, just for sort order - output.writeInt((int)encodeWeight(iterator.weight())); - writer.write(buffer, 0, output.getPosition()); - } - writer.close(); - new Sort().sort(tempInput, tempSorted); - reader = new Sort.ByteSequencesReader(tempSorted); - - PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true); - Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + TermFreqIterator iter = new WFSTTermFreqIteratorWrapper(iterator, + BytesRef.getUTF8SortedAsUnicodeComparator()); + IntsRef scratchInts = new IntsRef(); + BytesRef previous = null; + PositiveIntOutputs outputs = PositiveIntOutputs.getSingleton(true); + Builder builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + while ((scratch = iter.next()) != null) { + long cost = iter.weight(); - BytesRef previous = null; - BytesRef suggestion = new BytesRef(); - IntsRef scratchInts = new IntsRef(); - ByteArrayDataInput input = new ByteArrayDataInput(); - while (reader.read(scratch)) { - suggestion.bytes = scratch.bytes; - suggestion.offset = scratch.offset; - suggestion.length = scratch.length - 5; // int + separator - - input.reset(scratch.bytes); - input.skipBytes(suggestion.length + 1); // suggestion + separator - long cost = input.readInt(); - - if (previous == null) { - previous = new BytesRef(); - } else if (suggestion.equals(previous)) { - continue; // for duplicate suggestions, the best weight is actually added - } - Util.toIntsRef(suggestion, scratchInts); - builder.add(scratchInts, cost); - previous.copyBytes(suggestion); - } - fst = builder.finish(); - success = true; - } finally { - if (success) { - IOUtils.close(reader, writer); - } else { - IOUtils.closeWhileHandlingException(reader, writer); + if (previous == null) { + previous = new BytesRef(); + } else if (scratch.equals(previous)) { + continue; // for duplicate suggestions, the best weight is actually + // added } - - tempInput.delete(); - tempSorted.delete(); + Util.toIntsRef(scratch, scratchInts); + builder.add(scratchInts, cost); + previous.copyBytes(scratch); } + fst = builder.finish(); } @Override @@ -269,16 +227,10 @@ public class WFSTCompletionLookup extends Lookup { return output; } - @Override - public boolean add(CharSequence key, Object value) { - return false; // Not supported. - } - /** * Returns the weight associated with an input string, * or null if it does not exist. */ - @Override public Object get(CharSequence key) { Arc arc = new Arc(); Long result = null; @@ -288,20 +240,48 @@ public class WFSTCompletionLookup extends Lookup { if (result == null || !arc.isFinal()) { return null; } else { - return decodeWeight(result + arc.nextFinalOutput); + return Integer.valueOf(decodeWeight(result + arc.nextFinalOutput)); } } /** cost -> weight */ - private static float decodeWeight(long encoded) { - return Integer.MAX_VALUE - encoded; + private static int decodeWeight(long encoded) { + return (int)(Integer.MAX_VALUE - encoded); } /** weight -> cost */ - private static long encodeWeight(float value) { - if (Float.isNaN(value) || Float.isInfinite(value) || value < 0 || value > Integer.MAX_VALUE) { + private static int encodeWeight(long value) { + if (value < 0 || value > Integer.MAX_VALUE) { throw new UnsupportedOperationException("cannot encode value: " + value); } return Integer.MAX_VALUE - (int)value; } + + private final class WFSTTermFreqIteratorWrapper extends SortedTermFreqIteratorWrapper { + + WFSTTermFreqIteratorWrapper(TermFreqIterator source, + Comparator comparator) throws IOException { + super(source, comparator, true); + } + + @Override + protected void encode(ByteSequencesWriter writer, ByteArrayDataOutput output, byte[] buffer, BytesRef spare, long weight) throws IOException { + if (spare.length + 5 >= buffer.length) { + buffer = ArrayUtil.grow(buffer, spare.length + 5); + } + output.reset(buffer); + output.writeBytes(spare.bytes, spare.offset, spare.length); + output.writeByte((byte)0); // separator: not used, just for sort order + output.writeInt(encodeWeight(weight)); + writer.write(buffer, 0, output.getPosition()); + } + + @Override + protected long decode(BytesRef scratch, ByteArrayDataInput tmpInput) { + tmpInput.reset(scratch.bytes); + tmpInput.skipBytes(scratch.length - 4); // suggestion + separator + scratch.length -= 5; // sep + long + return tmpInput.readInt(); + } + } } diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java index 56a4751..c684164 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java @@ -55,24 +55,22 @@ public class JaspellLookup extends Lookup { final CharsRef charsSpare = new CharsRef(); while ((spare = tfit.next()) != null) { - float freq = tfit.weight(); + final long freq = tfit.weight(); if (spare.length == 0) { continue; } charsSpare.grow(spare.length); UnicodeUtil.UTF8toUTF16(spare.bytes, spare.offset, spare.length, charsSpare); - trie.put(charsSpare.toString(), new Float(freq)); + trie.put(charsSpare.toString(), weightAsNumber(freq)); } } - @Override public boolean add(CharSequence key, Object value) { trie.put(key, value); // XXX return false; } - @Override public Object get(CharSequence key) { return trie.get(key); } @@ -95,7 +93,7 @@ public class JaspellLookup extends Lookup { if (onlyMorePopular) { LookupPriorityQueue queue = new LookupPriorityQueue(num); for (String s : list) { - float freq = (Float)trie.get(s); + long freq = ((Number)trie.get(s)).longValue(); queue.insertWithOverflow(new LookupResult(new CharsRef(s), freq)); } for (LookupResult lr : queue.getResults()) { @@ -104,7 +102,7 @@ public class JaspellLookup extends Lookup { } else { for (int i = 0; i < maxCnt; i++) { String s = list.get(i); - float freq = (Float)trie.get(s); + long freq = ((Number)trie.get(s)).longValue(); res.add(new LookupResult(new CharsRef(s), freq)); } } @@ -131,7 +129,7 @@ public class JaspellLookup extends Lookup { node.splitchar = in.readChar(); byte mask = in.readByte(); if ((mask & HAS_VALUE) != 0) { - node.data = new Float(in.readFloat()); + node.data = weightAsNumber(in.readLong()); } if ((mask & LO_KID) != 0) { TSTNode kid = trie.new TSTNode('\0', node); @@ -171,7 +169,7 @@ public class JaspellLookup extends Lookup { if (node.data != null) mask |= HAS_VALUE; out.writeByte(mask); if (node.data != null) { - out.writeFloat((Float)node.data); + out.writeLong(((Number)node.data).longValue()); } writeRecursively(out, node.relatives[TSTNode.LOKID]); writeRecursively(out, node.relatives[TSTNode.EQKID]); diff --git a/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java b/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java index 56b00a3..d129bfe 100644 --- a/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java +++ b/modules/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java @@ -50,26 +50,24 @@ public class TSTLookup extends Lookup { } ArrayList tokens = new ArrayList(); - ArrayList vals = new ArrayList(); + ArrayList vals = new ArrayList(); BytesRef spare; CharsRef charsSpare = new CharsRef(); while ((spare = tfit.next()) != null) { charsSpare.grow(spare.length); UnicodeUtil.UTF8toUTF16(spare.bytes, spare.offset, spare.length, charsSpare); tokens.add(charsSpare.toString()); - vals.add(new Float(tfit.weight())); + vals.add(weightAsNumber(tfit.weight())); } autocomplete.balancedTree(tokens.toArray(), vals.toArray(), 0, tokens.size() - 1, root); } - @Override public boolean add(CharSequence key, Object value) { autocomplete.insert(root, key, value, 0); // XXX we don't know if a new node was created return true; } - @Override public Object get(CharSequence key) { List list = autocomplete.prefixCompletion(root, key, 0); if (list == null || list.isEmpty()) { @@ -107,7 +105,7 @@ public class TSTLookup extends Lookup { if (onlyMorePopular) { LookupPriorityQueue queue = new LookupPriorityQueue(num); for (TernaryTreeNode ttn : list) { - queue.insertWithOverflow(new LookupResult(ttn.token, (Float)ttn.val)); + queue.insertWithOverflow(new LookupResult(ttn.token, ((Number)ttn.val).longValue())); } for (LookupResult lr : queue.getResults()) { res.add(lr); @@ -115,7 +113,7 @@ public class TSTLookup extends Lookup { } else { for (int i = 0; i < maxCnt; i++) { TernaryTreeNode ttn = list.get(i); - res.add(new LookupResult(ttn.token, (Float)ttn.val)); + res.add(new LookupResult(ttn.token, ((Number)ttn.val).longValue())); } } return res; @@ -146,7 +144,7 @@ public class TSTLookup extends Lookup { node.token = in.readUTF(); } if ((mask & HAS_VALUE) != 0) { - node.val = new Float(in.readFloat()); + node.val = weightAsNumber(in.readLong()); } if ((mask & LO_KID) != 0) { node.loKid = new TernaryTreeNode(); @@ -184,7 +182,7 @@ public class TSTLookup extends Lookup { if (node.val != null) mask |= HAS_VALUE; out.writeByte(mask); if (node.token != null) out.writeUTF(node.token); - if (node.val != null) out.writeFloat((Float)node.val); + if (node.val != null) out.writeLong(((Number)node.val).longValue()); // recurse and write kids if (node.loKid != null) { writeRecursively(out, node.loKid); diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java index a2deec4..73f5ae8 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/PersistenceTest.java @@ -17,8 +17,10 @@ package org.apache.lucene.search.suggest; import java.io.File; +import java.util.List; import org.apache.lucene.search.suggest.Lookup; +import org.apache.lucene.search.suggest.Lookup.LookupResult; import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; import org.apache.lucene.search.suggest.jaspell.JaspellLookup; import org.apache.lucene.search.suggest.tst.TSTLookup; @@ -74,16 +76,18 @@ public class PersistenceTest extends LuceneTestCase { lookup.load(storeDir); // Assert validity. - float previous = Float.NEGATIVE_INFINITY; + long previous = Long.MIN_VALUE; for (TermFreq k : keys) { - Float val = (Float) lookup.get(_TestUtil.bytesToCharSequence(k.term, random)); - assertNotNull(k.term.utf8ToString(), val); + List list = lookup.lookup(_TestUtil.bytesToCharSequence(k.term, random), false, 1); + assertEquals(1, list.size()); + LookupResult lookupResult = list.get(0); + assertNotNull(k.term.utf8ToString(), lookupResult.key); if (supportsExactWeights) { - assertEquals(k.term.utf8ToString(), Float.valueOf(k.v), val); + assertEquals(k.term.utf8ToString(), k.v, lookupResult.value); } else { - assertTrue(val + ">=" + previous, val >= previous); - previous = val.floatValue(); + assertTrue(lookupResult.value + ">=" + previous, lookupResult.value >= previous); + previous = lookupResult.value; } } } diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java index 8195281..92b1947 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/TestBytesRefList.java @@ -82,6 +82,5 @@ public class TestBytesRefList extends LuceneTestCase { assertEquals("entry " + i + " doesn't match", stringList.get(i), spare.utf8ToString()); } - } } diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java index 6e74bc2..5638894 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/TestTermFreqIterator.java @@ -17,12 +17,16 @@ package org.apache.lucene.search.suggest; * the License. */ +import java.util.Comparator; import java.util.Iterator; import java.util.Map; import java.util.TreeMap; import org.apache.lucene.search.spell.TermFreqIterator; +import org.apache.lucene.store.ByteArrayDataOutput; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefHash; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -38,7 +42,8 @@ public class TestTermFreqIterator extends LuceneTestCase { public void testTerms() throws Exception { int num = atLeast(10000); - TreeMap sorted = new TreeMap(); + Comparator comparator = random.nextBoolean() ? BytesRef.getUTF8SortedAsUnicodeComparator() : BytesRef.getUTF8SortedAsUTF16Comparator(); + TreeMap sorted = new TreeMap(comparator); TermFreq[] unsorted = new TermFreq[num]; for (int i = 0; i < num; i++) { @@ -52,13 +57,13 @@ public class TestTermFreqIterator extends LuceneTestCase { } // test the sorted iterator wrapper - TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), BytesRef.getUTF8SortedAsUnicodeComparator()); + TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), comparator); Iterator> expected = sorted.entrySet().iterator(); while (expected.hasNext()) { Map.Entry entry = expected.next(); assertEquals(entry.getKey(), wrapper.next()); - assertEquals(entry.getValue().longValue(), wrapper.weight(), 0F); + assertEquals(entry.getValue().longValue(), wrapper.weight()); } assertNull(wrapper.next()); @@ -72,4 +77,57 @@ public class TestTermFreqIterator extends LuceneTestCase { } assertEquals(sorted, actual); } + + + public void testRaw() throws Exception { + int num = atLeast(10000); + + Comparator comparator = BytesRef.getUTF8SortedAsUnicodeComparator(); + BytesRefHash sorted = new BytesRefHash(); + TermFreq[] unsorted = new TermFreq[num]; + byte[] buffer = new byte[0]; + ByteArrayDataOutput output = new ByteArrayDataOutput(buffer); + + for (int i = 0; i < num; i++) { + BytesRef spare; + long weight; + do { + spare = new BytesRef(_TestUtil.randomUnicodeString(random)); + if (spare.length + 8 >= buffer.length) { + buffer = ArrayUtil.grow(buffer, spare.length + 8); + } + output.reset(buffer); + output.writeBytes(spare.bytes, spare.offset, spare.length); + weight = random.nextLong(); + output.writeLong(weight); + + } while (sorted.add(new BytesRef(buffer, 0, output.getPosition())) < 0); + unsorted[i] = new TermFreq(spare, weight); + } + + // test the sorted iterator wrapper + TermFreqIterator wrapper = new SortedTermFreqIteratorWrapper(new TermFreqArrayIterator(unsorted), comparator, true); + int[] sort = sorted.sort(comparator); + int size = sorted.size(); + BytesRef spare = new BytesRef(); + for (int i = 0; i < size; i++) { + sorted.get(sort[i], spare); + spare.length -= 8; // sub the long value + assertEquals(spare, wrapper.next()); + spare.offset = spare.offset + spare.length; + spare.length = 8; + assertEquals(asLong(spare), wrapper.weight()); + } + assertNull(wrapper.next()); + } + + public static long asLong(BytesRef b) { + return (((long) asIntInternal(b, b.offset) << 32) | asIntInternal(b, + b.offset + 4) & 0xFFFFFFFFL); + } + + private static int asIntInternal(BytesRef b, int pos) { + return ((b.bytes[pos++] & 0xFF) << 24) | ((b.bytes[pos++] & 0xFF) << 16) + | ((b.bytes[pos++] & 0xFF) << 8) | (b.bytes[pos] & 0xFF); + } } diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java index f97d6b5..339282e 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java @@ -165,9 +165,9 @@ public class FSTCompletionTest extends LuceneTestCase { // All the weights were constant, so all returned buckets must be constant, whatever they // are. - Float previous = null; + Long previous = null; for (TermFreq tf : keys) { - Float current = (Float)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random)); + Long current = ((Number)lookup.get(_TestUtil.bytesToCharSequence(tf.term, random))).longValue(); if (previous != null) { assertEquals(previous, current); } @@ -181,7 +181,7 @@ public class FSTCompletionTest extends LuceneTestCase { FSTCompletionLookup lookup = new FSTCompletionLookup(); lookup.build(new TermFreqArrayIterator(input)); for (TermFreq tf : input) { - assertTrue("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random)) != null); + assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random))); assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random), true, 1).get(0).key.toString()); } diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FloatMagicTest.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FloatMagicTest.java deleted file mode 100644 index 2129142..0000000 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/FloatMagicTest.java +++ /dev/null @@ -1,140 +0,0 @@ -package org.apache.lucene.search.suggest.fst; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.*; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.NumericUtils; -import org.junit.Ignore; -import org.junit.Test; - -public class FloatMagicTest extends LuceneTestCase { - public void testFloatMagic() { - ArrayList floats = new ArrayList(Arrays.asList( - Float.intBitsToFloat(0x7f800001), // NaN (invalid combination). - Float.intBitsToFloat(0x7fffffff), // NaN (invalid combination). - Float.intBitsToFloat(0xff800001), // NaN (invalid combination). - Float.intBitsToFloat(0xffffffff), // NaN (invalid combination). - Float.POSITIVE_INFINITY, - Float.MAX_VALUE, - 100f, - 0f, - 0.1f, - Float.MIN_VALUE, - Float.NaN, - -0.0f, - -Float.MIN_VALUE, - -0.1f, - -1f, - -10f, - Float.NEGATIVE_INFINITY)); - - // Sort them using juc. - Collections.sort(floats); - - // Convert to sortable int4 representation (as long to have an unsigned sort). - long [] int4 = new long [floats.size()]; - for (int i = 0; i < floats.size(); i++) { - int4[i] = FloatMagic.toSortable(floats.get(i)) & 0xffffffffL; - - /* - System.out.println( - String.format("raw %8s sortable %8s %8s numutils %8s %s", - Integer.toHexString(Float.floatToRawIntBits(floats.get(i))), - Integer.toHexString(FloatMagic.toSortable(floats.get(i))), - Integer.toHexString(FloatMagic.unsignedOrderedToFloatBits(FloatMagic.toSortable(floats.get(i)))), - Integer.toHexString(NumericUtils.floatToSortableInt(floats.get(i))), - floats.get(i))); - */ - } - - // Sort and compare. Should be identical order. - Arrays.sort(int4); - ArrayList backFromFixed = new ArrayList(); - for (int i = 0; i < int4.length; i++) { - backFromFixed.add(FloatMagic.fromSortable((int) int4[i])); - } - - /* - for (int i = 0; i < int4.length; i++) { - System.out.println( - floats.get(i) + " " + FloatMagic.fromSortable((int) int4[i])); - } - */ - - assertEquals(floats, backFromFixed); - } - - @Ignore("Once checked, valid forever?") @Test - public void testRoundTripFullRange() { - int i = 0; - do { - float f = Float.intBitsToFloat(i); - float f2 = FloatMagic.fromSortable(FloatMagic.toSortable(f)); - - if (!((Float.isNaN(f) && Float.isNaN(f2)) || f == f2)) { - throw new RuntimeException("! " + Integer.toHexString(i) + "> " + f + " " + f2); - } - - if ((i & 0xffffff) == 0) { - System.out.println(Integer.toHexString(i)); - } - - i++; - } while (i != 0); - } - - @Ignore("Once checked, valid forever?") @Test - public void testIncreasingFullRange() { - // -infinity ... -0.0 - for (int i = 0xff800000; i != 0x80000000; i--) { - checkSmaller(i, i - 1); - } - - // -0.0 +0.0 - checkSmaller(0x80000000, 0); - - // +0.0 ... +infinity - for (int i = 0; i != 0x7f800000; i++) { - checkSmaller(i, i + 1); - } - - // All other are NaNs and should be after positive infinity. - final long infinity = toSortableL(Float.POSITIVE_INFINITY); - for (int i = 0x7f800001; i != 0x7fffffff; i++) { - assertTrue(infinity < toSortableL(Float.intBitsToFloat(i))); - } - for (int i = 0xff800001; i != 0xffffffff; i++) { - assertTrue(infinity < toSortableL(Float.intBitsToFloat(i))); - } - } - - private long toSortableL(float f) { - return FloatMagic.toSortable(f) & 0xffffffffL; - } - - private void checkSmaller(int i1, int i2) { - float f1 = Float.intBitsToFloat(i1); - float f2 = Float.intBitsToFloat(i2); - if (f1 > f2) { - throw new AssertionError(f1 + " " + f2 + " " + i1 + " " + i2); - } - assertTrue(toSortableL(f1) < toSortableL(f2)); - } -} diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java index f4f9853..3a7937c 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/TestSort.java @@ -20,6 +20,7 @@ package org.apache.lucene.search.suggest.fst; import java.io.*; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import org.apache.lucene.search.suggest.fst.Sort.BufferSize; import org.apache.lucene.search.suggest.fst.Sort.ByteSequencesWriter; @@ -61,7 +62,7 @@ public class TestSort extends LuceneTestCase { @Test public void testIntermediateMerges() throws Exception { // Sort 20 mb worth of data with 1mb buffer, binary merging. - SortInfo info = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), 2), + SortInfo info = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), 2), generateRandom(Sort.MB * 20)); assertTrue(info.mergeRounds > 10); } @@ -69,7 +70,7 @@ public class TestSort extends LuceneTestCase { @Test public void testSmallRandom() throws Exception { // Sort 20 mb worth of data with 1mb buffer. - SortInfo sortInfo = checkSort(new Sort(BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), + SortInfo sortInfo = checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(1), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), generateRandom(Sort.MB * 20)); assertEquals(1, sortInfo.mergeRounds); } @@ -77,7 +78,7 @@ public class TestSort extends LuceneTestCase { @Test @Nightly public void testLargerRandom() throws Exception { // Sort 100MB worth of data with 15mb buffer. - checkSort(new Sort(BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), + checkSort(new Sort(Sort.DEFAULT_COMPARATOR, BufferSize.megabytes(16), Sort.defaultTempDir(), Sort.MAX_TEMPFILES), generateRandom(Sort.MB * 100)); } @@ -92,14 +93,25 @@ public class TestSort extends LuceneTestCase { byte [][] bytes = data.toArray(new byte[data.size()][]); return bytes; } - + + static final Comparator unsignedByteOrderComparator = new Comparator() { + public int compare(byte[] left, byte[] right) { + final int max = Math.min(left.length, right.length); + for (int i = 0, j = 0; i < max; i++, j++) { + int diff = (left[i] & 0xff) - (right[j] & 0xff); + if (diff != 0) + return diff; + } + return left.length - right.length; + } + }; /** * Check sorting data on an instance of {@link Sort}. */ private SortInfo checkSort(Sort sort, byte[][] data) throws IOException { File unsorted = writeAll("unsorted", data); - Arrays.sort(data, Sort.unsignedByteOrderComparator); + Arrays.sort(data, unsignedByteOrderComparator); File golden = writeAll("golden", data); File sorted = new File(tempDir, "sorted"); diff --git a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java index 916eeb9..6cadef3 100644 --- a/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java +++ b/modules/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java @@ -117,7 +117,7 @@ public class WFSTCompletionTest extends LuceneTestCase { // TODO: could be faster... but its slowCompletor for a reason for (Map.Entry e : slowCompletor.entrySet()) { if (e.getKey().startsWith(prefix)) { - matches.add(new LookupResult(e.getKey(), (float)e.getValue().longValue())); + matches.add(new LookupResult(e.getKey(), e.getValue().longValue())); } } diff --git a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java index 8f0cade..525ce3b 100644 --- a/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java +++ b/solr/core/src/java/org/apache/solr/spelling/suggest/Suggester.java @@ -153,11 +153,6 @@ public class Suggester extends SolrSpellChecker { build(core, searcher); } - public void add(CharsRef query, int numHits) { - LOG.info("add " + query + ", " + numHits); - lookup.add(query, new Integer(numHits)); - } - static SpellingResult EMPTY_RESULT = new SpellingResult(); @Override @@ -182,7 +177,7 @@ public class Suggester extends SolrSpellChecker { Collections.sort(suggestions); } for (LookupResult lr : suggestions) { - res.add(t, lr.key.toString(), ((Number)lr.value).intValue()); + res.add(t, lr.key.toString(), (int)lr.value); } } return res;