new {Long|Int}TrieRangeFilter(....).asQuery().
+ * This is a synonym for wrapping with a {@link ConstantScoreQuery},
+ * but this query returns a better toString() variant.
+ */
+ public Query asQuery() {
+ return new ConstantScoreQuery(this) {
+
+ /** this instance return a nicer String variant than the original {@link ConstantScoreQuery} */
+ //@Override
+ public String toString(final String field) {
+ // return a more convenient representation of this query than ConstantScoreQuery does:
+ return ((AbstractTrieRangeFilter) filter).toString(field)+ToStringUtils.boost(getBoost());
+ }
+
+ };
+ }
+
+ void fillBits(
+ final IndexReader reader,
+ final OpenBitSet bits, final TermDocs termDocs,
+ String field,
+ final String lowerTerm, final String upperTerm
+ ) throws IOException {
+ final int len=lowerTerm.length();
+ assert upperTerm.length()==len;
+ field=field.intern();
+
+ // find the docs
+ final TermEnum enumerator = reader.terms(new Term(field, lowerTerm));
+ try {
+ do {
+ final Term term = enumerator.term();
+ if (term!=null && term.field()==field) {
+ // break out when upperTerm reached or length of term is different
+ final String t=term.text();
+ if (len!=t.length() || t.compareTo(upperTerm)>0) break;
+ // we have a good term, find the docs
+ lastNumberOfTerms++;
+ termDocs.seek(enumerator);
+ while (termDocs.next()) bits.set(termDocs.doc());
+ } else break;
+ } while (enumerator.next());
+ } finally {
+ enumerator.close();
+ }
+ }
+
+ // members
+ final String[] fields;
+ final int precisionStep;
+ final Number min,max;
+ final boolean minInclusive,maxInclusive;
+
+ private int lastNumberOfTerms=-1;
+}
Index: contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java (revision 0)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieRangeFilter.java (revision 0)
@@ -0,0 +1,140 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.util.OpenBitSet;
+
+
+/**
+ * Implementation of a Lucene {@link Filter} that implements trie-based range filtering for ints/floats.
+ * This filter depends on a specific structure of terms in the index that can only be created
+ * by {@link TrieUtils} methods.
+ * For more information, how the algorithm works, see the {@linkplain org.apache.lucene.search.trie package description}.
+ */
+public class IntTrieRangeFilter extends AbstractTrieRangeFilter {
+
+ /**
+ * A trie filter for matching trie coded values using the given field name and
+ * the default helper field.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
+ * This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String[])}. + *
This is the recommended usage of TrieUtils/IntTrieRangeFilter.
+ */
+ public IntTrieRangeFilter(final String field, final int precisionStep,
+ final Integer min, final Integer max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ this(
+ new String[]{field, field+TrieUtils.LOWER_PRECISION_FIELD_NAME_SUFFIX},
+ precisionStep,min,max,minInclusive,maxInclusive
+ );
+ }
+
+ /**
+ * Expert: A trie filter for matching trie coded values using the given field names.
+ * You can specify the main and helper field name, that was used to idex the values.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
+ *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
+ */
+ public IntTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
+ final Integer min, final Integer max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
+ }
+
+ /**
+ * Expert: A trie filter for matching trie coded values
+ * using the given field names. If the array of field names is shorter than the
+ * trieCoded one, all trieCoded values with higher index get the last field name.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
+ *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}.
+ */
+ public IntTrieRangeFilter(final String[] fields, final int precisionStep,
+ Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ super(fields, precisionStep, min, max, minInclusive, maxInclusive);
+ }
+
+ /**
+ * Returns a DocIdSet that provides the documents which should be permitted or prohibited in search results.
+ */
+ //@Override
+ public DocIdSet getDocIdSet(final IndexReader reader) throws IOException {
+ // calculate the upper and lower bounds respecting the inclusive and null values.
+ int minBound=(this.min==null) ? Integer.MIN_VALUE : (
+ minInclusive ? this.min.intValue() : (this.min.intValue()+1)
+ );
+ int maxBound=(this.max==null) ? Integer.MAX_VALUE : (
+ maxInclusive ? this.max.intValue() : (this.max.intValue()-1)
+ );
+
+ resetLastNumberOfTerms();
+ if (minBound > maxBound) {
+ // shortcut, no docs will match this
+ return DocIdSet.EMPTY_DOCIDSET;
+ } else {
+ final OpenBitSet bits = new OpenBitSet(reader.maxDoc());
+ final TermDocs termDocs = reader.termDocs();
+ try {
+ TrieUtils.splitIntRange(new TrieUtils.IntRangeBuilder() {
+
+ //@Override
+ public final void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+ try {
+ fillBits(
+ reader, bits, termDocs,
+ fields[Math.min(fields.length-1, level)],
+ minPrefixCoded, maxPrefixCoded
+ );
+ } catch (IOException ioe) {
+ // IntRangeBuilder is not allowed to throw checked exceptions:
+ // wrap as RuntimeException
+ throw new RuntimeException(ioe);
+ }
+ }
+
+ }, precisionStep, minBound, maxBound);
+ } catch (RuntimeException e) {
+ if (e.getCause() instanceof IOException) throw (IOException)e.getCause();
+ throw e;
+ } finally {
+ termDocs.close();
+ }
+ return bits;
+ }
+ }
+
+}
Index: contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java (revision 0)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java (revision 0)
@@ -0,0 +1,140 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.search.Filter;
+import org.apache.lucene.search.DocIdSet;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.util.OpenBitSet;
+
+
+/**
+ * Implementation of a Lucene {@link Filter} that implements trie-based range filtering for longs/doubles.
+ * This filter depends on a specific structure of terms in the index that can only be created
+ * by {@link TrieUtils} methods.
+ * For more information, how the algorithm works, see the {@linkplain org.apache.lucene.search.trie package description}.
+ */
+public class LongTrieRangeFilter extends AbstractTrieRangeFilter {
+
+ /**
+ * A trie filter for matching trie coded values using the given field name and
+ * the default helper field.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
+ *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String[])}. + *
This is the recommended usage of TrieUtils/LongTrieRangeFilter.
+ */
+ public LongTrieRangeFilter(final String field, final int precisionStep,
+ final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ this(
+ new String[]{field, field+TrieUtils.LOWER_PRECISION_FIELD_NAME_SUFFIX},
+ precisionStep,min,max,minInclusive,maxInclusive
+ );
+ }
+
+ /**
+ * Expert: A trie filter for matching trie coded values using the given field names.
+ * You can specify the main and helper field name, that was used to idex the values.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
+ *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
+ */
+ public LongTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
+ final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
+ }
+
+ /**
+ * Expert: A trie filter for matching trie coded values
+ * using the given field names. If the array of field names is shorter than the
+ * trieCoded one, all trieCoded values with higher index get the last field name.
+ * precisionStep must me equal or a multiple of the precisionStep
+ * used for indexing the values.
+ * You can leave the bounds open, by supplying null for min and/or
+ * max. Inclusive/exclusive bounds can also be supplied.
+ * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
+ *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}. + */ + public LongTrieRangeFilter(final String[] fields, final int precisionStep, + Long min, Long max, final boolean minInclusive, final boolean maxInclusive + ) { + super(fields, precisionStep, min, max, minInclusive, maxInclusive); + } + + /** + * Returns a DocIdSet that provides the documents which should be permitted or prohibited in search results. + */ + //@Override + public DocIdSet getDocIdSet(final IndexReader reader) throws IOException { + // calculate the upper and lower bounds respecting the inclusive and null values. + long minBound=(this.min==null) ? Long.MIN_VALUE : ( + minInclusive ? this.min.longValue() : (this.min.longValue()+1L) + ); + long maxBound=(this.max==null) ? Long.MAX_VALUE : ( + maxInclusive ? this.max.longValue() : (this.max.longValue()-1L) + ); + + resetLastNumberOfTerms(); + if (minBound > maxBound) { + // shortcut, no docs will match this + return DocIdSet.EMPTY_DOCIDSET; + } else { + final OpenBitSet bits = new OpenBitSet(reader.maxDoc()); + final TermDocs termDocs = reader.termDocs(); + try { + TrieUtils.splitLongRange(new TrieUtils.LongRangeBuilder() { + + //@Override + public final void addRange(String minPrefixCoded, String maxPrefixCoded, int level) { + try { + fillBits( + reader, bits, termDocs, + fields[Math.min(fields.length-1, level)], + minPrefixCoded, maxPrefixCoded + ); + } catch (IOException ioe) { + // LongRangeBuilder is not allowed to throw checked exceptions: + // wrap as RuntimeException + throw new RuntimeException(ioe); + } + } + + }, precisionStep, minBound, maxBound); + } catch (RuntimeException e) { + if (e.getCause() instanceof IOException) throw (IOException)e.getCause(); + throw e; + } finally { + termDocs.close(); + } + return bits; + } + } + +} Index: contrib/queries/src/java/org/apache/lucene/search/trie/package.html =================================================================== --- contrib/queries/src/java/org/apache/lucene/search/trie/package.html (revision 742672) +++ contrib/queries/src/java/org/apache/lucene/search/trie/package.html (working copy) @@ -1,7 +1,7 @@
-This package provides fast numeric range queries/filters on long, double or Date
-fields based on trie structures.
This package provides fast numeric range queries/filters on long, double, int,
+or float (and other data types, that can be converted to numerical values) fields based on trie structures.
See the publication about panFMP, where this algorithm was described: @@ -14,66 +14,96 @@ it cannot handle numerical ranges (e.g., field value is inside user defined bounds, even dates are numerical values). We have developed an extension to Apache Lucene that stores the numerical values in a special string-encoded format with variable precision -(all numerical values like doubles, longs, and timestamps are converted to lexicographic sortable string representations -and stored with different precisions from one byte to the full 8 bytes - depending on the variant used). -For a more detailed description of how the values are stored, see {@link org.apache.lucene.search.trie.TrieUtils}. -A range is then divided recursively into multiple intervals for searching: -The center of the range is searched only with the lowest possible precision in the trie, while the boundaries are matched -more exactly. This reduces the number of terms dramatically.
+(all numerical values like doubles, longs, floats, and ints are converted to lexicographic sortable string representations +and stored with different precisions. For a more detailed description of how the values are stored, +see {@link org.apache.lucene.search.trie.TrieUtils}. A range is then divided recursively into +multiple intervals for searching: +The center of the range is searched only with the lowest possible precision in the trie, +while the boundaries are matched more exactly. This reduces the number of terms dramatically. -For the variant that uses a lowest precision of 1-byte the index -contains only a maximum of 256 distinct values in the lowest precision. -Overall, a range could consist of a theoretical maximum of +
For the variant that stores long values in 8 different precisions (each reduced by 8 bits) that
+uses a lowest precision of 1 byte, the index contains only a maximum of 256 distinct values in the
+lowest precision. Overall, a range could consist of a theoretical maximum of
7*255*2 + 255 = 3825 distinct terms (when there is a term for every distinct value of an
-8-byte-number in the index and the range covers all of them; a maximum of 255 distinct values is used
+8-byte-number in the index and the range covers almost all of them; a maximum of 255 distinct values is used
because it would always be possible to reduce the full 256 values to one term with degraded precision).
In practise, we have seen up to 300 terms in most cases (index with 500,000 metadata records
and a uniform value distribution).
There are two other variants of encoding: 4bit and 2bit. Each variant stores more different precisions
-of the longs and thus needs more storage space (because it generates more and longer terms -
-4bit: two times the length and number of terms; 2bit: four times the length and number of terms).
-But on the other hand, the maximum number of distinct terms used for range queries is
-15*15*2 + 15 = 465 for the 4bit variant, and
-31*3*2 + 3 = 189 for the 2bit variant.
You can choose any precisionStep when encoding integer values.
+Lower step values mean more precisions and so more terms in index (and index gets larger).
+On the other hand, the maximum number of terms to match reduces, which optimized query speed.
+The formula to calculate the maximum term count is:
+
+ n = [ (bitsPerValue/precisionStep - 1) * (2^precisionStep - 1 ) * 2 ] + (2^precisionStep - 1 ) ++
(this formula is only correct, when bitsPerValue/precisionStep is an integer;
+in other cases, the value must be rounded up and the last summand must contain the modulo of the division as
+precision step).
+For longs stored using a precision step of 4, n = 15*15*2 + 15 = 465, and for a precision
+step of 2, n = 31*3*2 + 3 = 189. But the faster search speed is reduced by more seeking
+in the term enum of the index. Because of this, the ideal precisionStep value can only
+be found out by testing. Important: You can index with a lower precision value and test search speed
+using a multiple of the original step value.
This dramatically improves the performance of Apache Lucene with range queries, which are no longer dependent on the index size and the number of distinct values because there is an upper limit unrelated to either of these properties.
To use the new query types the numerical values, which may be long, double or Date,
-the values must be stored during indexing in a special format in the index (using {@link org.apache.lucene.search.trie.TrieUtils}).
-This can be done like this:
To use the new query types the numerical values, which may belong, double, int,
+float, or Date, the values must be stored during indexing in a special format in the index
+(using {@link org.apache.lucene.search.trie.TrieUtils}). This can be done like this:
+ // chose a step value, 8 is a general good value for large indexes:
+ int precisionStep = 8;
+
Document doc = new Document();
- // add some standard fields:
+
+ // add some standard fields:
String svalue = "anything to index";
- doc.add(new Field("exampleString", svalue, Field.Store.YES, Field.Index.ANALYZED) ;
- // add some numerical fields:
- double fvalue = 1.057E17;
- TrieUtils.VARIANT_8BIT.addDoubleTrieCodedDocumentField(doc, "exampleDouble", fvalue, true /* index the field */, Field.Store.YES);
+ doc.add(new Field("exampleString", svalue, Field.Store.YES, Field.Index.ANALYZED));
+
+ // add some numerical fields:
long lvalue = 121345L;
- TrieUtils.VARIANT_8BIT.addLongTrieCodedDocumentField(doc, "exampleLong", lvalue, true /* index the field */, Field.Store.YES);
- Date dvalue = new Date(); // actual time
- TrieUtils.VARIANT_8BIT.addDateTrieCodedDocumentField(doc, "exampleDate", dvalue, true /* index the field */, Field.Store.YES);
- // add document to IndexWriter
+ TrieUtils.addIndexedFields(doc, "exampleLong", TrieUtils.trieCodeLong(lvalue, precisionStep));
+ double dvalue = 1.057E17;
+ TrieUtils.addIndexedFields(doc, "exampleDouble", TrieUtils.trieCodeLong(TrieUtils.doubleToSortableLong(dvalue), precisionStep));
+ int ivalue = 121345;
+ TrieUtils.addIndexedFields(doc, "exampleInt", TrieUtils.trieCodeInt(ivalue, precisionStep));
+ float fvalue = 1.057E17f;
+ TrieUtils.addIndexedFields(doc, "exampleFloat", TrieUtils.trieCodeInt(TrieUtils.floatToSortableInt(fvalue), precisionStep));
+ Date datevalue = new Date(); // actual time
+ TrieUtils.addIndexedFields(doc, "exampleDate", TrieUtils.trieCodeLong(datevalue.getTime(), precisionStep));
+
+ // if you want to also store one of the values:
+ doc.add(new Field("exampleLong", Long.toString(lvalue), Field.Store.YES, Field.Index.NO));
+
+ // or as encoded value:
+ doc.add(new Field("exampleLong2", TrieUtils.longToPrefixCoded(lvalue), Field.Store.YES, Field.Index.NO));
+
+ // now add document to IndexWriter, as usual
-The numeric index fields you prepared in this way can be searched by {@link org.apache.lucene.search.trie.TrieRangeQuery}:
+The numeric index fields you prepared in this way can be searched by +{@link org.apache.lucene.search.trie.LongTrieRangeFilter} or {@link org.apache.lucene.search.trie.IntTrieRangeFilter}:
- // Java 1.4, because Double.valueOf(double) is not available:
- Query q = new TrieRangeQuery("exampleDouble", new Double(1.0E17), new Double(2.0E17), TrieUtils.VARIANT_8BIT);
- // OR, Java 1.5, using autoboxing:
- Query q = new TrieRangeQuery("exampleDouble", 1.0E17, 2.0E17, TrieUtils.VARIANT_8BIT);
+ // Java 1.4, because Double.valueOf(double) is not available:
+ Query q = new LongTrieRangeFilter("exampleLong", precisionStep, new Long(123L), new Long(999999L), true, true).asQuery();
+
+ // OR, Java 1.5, using autoboxing:
+ Query q = new LongTrieRangeFilter("exampleDouble", precisionStep, 123L, 999999L, true, true).asQuery();
+
+ // execute the search, as usual:
TopDocs docs = searcher.search(q, 10);
for (int i = 0; i<docs.scoreDocs.length; i++) {
Document doc = searcher.doc(docs.scoreDocs[i].doc);
System.out.println(doc.get("exampleString"));
- // decode the stored numerical value (important!!!):
- System.out.println(TrieUtils.VARIANT_8BIT.trieCodedToDouble(doc.get("exampleDouble")));
+
+ // decode a prefix coded, stored field:
+ System.out.println(TrieUtils.prefixCodedToLong(doc.get("exampleLong2")));
}
@@ -82,9 +112,9 @@
Comparisions of the different types of RangeQueries on an index with about 500,000 docs showed
that the old {@link org.apache.lucene.search.RangeQuery} (with raised
{@link org.apache.lucene.search.BooleanQuery} clause count) took about 30-40 secs to complete,
-{@link org.apache.lucene.search.ConstantScoreRangeQuery} took 5 secs and
-{@link org.apache.lucene.search.trie.TrieRangeQuery} took <100ms to
-complete (on an Opteron64 machine, Java 1.5, {@link org.apache.lucene.search.trie.TrieUtils#VARIANT_8BIT}).
+{@link org.apache.lucene.search.ConstantScoreRangeQuery} took 5 secs and executing
+{@link org.apache.lucene.search.trie.LongTrieRangeFilter}.asQuery() took <100ms to
+complete (on an Opteron64 machine, Java 1.5, 8 bit precision step).
This query type was developed for a geographic portal, where the performance for
e.g. bounding boxes or exact date/time stamps is important.
min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeFilter(final String field, String min, String max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- if (min==null && max==null) throw new IllegalArgumentException("The min and max values cannot be both null.");
- this.trieVariant=variant;
- this.field=field.intern();
- // just for toString()
- this.minUnconverted=min;
- this.maxUnconverted=max;
- this.minInclusive=minInclusive;
- this.maxInclusive=maxInclusive;
- // encode bounds
- this.min=(min==null) ? trieVariant.TRIE_CODED_NUMERIC_MIN : (
- minInclusive ? min : variant.incrementTrieCoded(min)
- );
- this.max=(max==null) ? trieVariant.TRIE_CODED_NUMERIC_MAX : (
- maxInclusive ? max : variant.decrementTrieCoded(max)
- );
- // check encoded values
- if (
- this.min.length() != trieVariant.TRIE_CODED_LENGTH ||
- this.max.length() != trieVariant.TRIE_CODED_LENGTH
- ) throw new NumberFormatException("Invalid trie encoded numerical value representation (incompatible length).");
- }
-
- /**
- * Generates a trie filter using the supplied field with range bounds in numeric form (double).
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeFilter(final String field, final Double min, final Double max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- this(
- field,
- (min==null) ? null : variant.doubleToTrieCoded(min.doubleValue()),
- (max==null) ? null : variant.doubleToTrieCoded(max.doubleValue()),
- minInclusive,
- maxInclusive,
- variant
- );
- this.minUnconverted=min;
- this.maxUnconverted=max;
- }
-
- /**
- * Generates a trie filter using the supplied field with range bounds in date/time form.
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeFilter(final String field, final Date min, final Date max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- this(
- field,
- (min==null) ? null : variant.dateToTrieCoded(min),
- (max==null) ? null : variant.dateToTrieCoded(max),
- minInclusive,
- maxInclusive,
- variant
- );
- this.minUnconverted=min;
- this.maxUnconverted=max;
- }
-
- /**
- * Generates a trie filter using the supplied field with range bounds in integer form (long).
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeFilter(final String field, final Long min, final Long max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- this(
- field,
- (min==null) ? null : variant.longToTrieCoded(min.longValue()),
- (max==null) ? null : variant.longToTrieCoded(max.longValue()),
- minInclusive,
- maxInclusive,
- variant
- );
- this.minUnconverted=min;
- this.maxUnconverted=max;
- }
-
- //@Override
- public String toString() {
- return toString(null);
- }
-
- public String toString(final String field) {
- final StringBuffer sb=new StringBuffer();
- if (!this.field.equals(field)) sb.append(this.field).append(':');
- return sb.append(minInclusive ? '[' : '{')
- .append((minUnconverted==null) ? "*" : minUnconverted.toString())
- .append(" TO ")
- .append((maxUnconverted==null) ? "*" : maxUnconverted.toString())
- .append(maxInclusive ? ']' : '}').toString();
- }
-
- /**
- * Two instances are equal if they have the same trie-encoded range bounds, same field, and same variant.
- * If one of the instances uses an exclusive lower bound, it is equal to a range with inclusive bound,
- * when the inclusive lower bound is equal to the incremented exclusive lower bound of the other one.
- * The same applys for the upper bound in other direction.
- */
- //@Override
- public final boolean equals(final Object o) {
- if (o instanceof TrieRangeFilter) {
- TrieRangeFilter q=(TrieRangeFilter)o;
- // trieVariants are singleton per type, so no equals needed.
- return (field==q.field && min.equals(q.min) && max.equals(q.max) && trieVariant==q.trieVariant);
- } else return false;
- }
-
- //@Override
- public final int hashCode() {
- // the hash code uses from the variant only the number of bits, as this is unique for the variant
- return field.hashCode()+(min.hashCode()^0x14fa55fb)+(max.hashCode()^0x733fa5fe)+(trieVariant.TRIE_BITS^0x64365465);
- }
-
- /** prints the String in hexadecimal \\u notation (for debugging of setBits()) */
- private String stringToHexDigits(final String s) {
- StringBuffer sb=new StringBuffer(s.length()*3);
- for (int i=0,c=s.length(); iThis class wraps a {@link TrieRangeFilter}.
- * @see TrieRangeFilter
- */
-public final class TrieRangeQuery extends ConstantScoreQuery {
-
- /**
- * Universal constructor (expert use only): Uses already trie-converted min/max values.
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeQuery(final String field, final String min, final String max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- super(new TrieRangeFilter(field,min,max,minInclusive,maxInclusive,variant));
- }
-
- /**
- * A trie query using the supplied field with range bounds in numeric form (double).
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeQuery(final String field, final Double min, final Double max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- super(new TrieRangeFilter(field,min,max,minInclusive,maxInclusive,variant));
- }
-
- /**
- * A trie query using the supplied field with range bounds in date/time form.
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeQuery(final String field, final Date min, final Date max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- super(new TrieRangeFilter(field,min,max,minInclusive,maxInclusive,variant));
- }
-
- /**
- * A trie query using the supplied field with range bounds in integer form (long).
- * You can set min or max (but not both) to null to leave one bound open.
- * With minInclusive and maxInclusive can be choosen, if the corresponding
- * bound should be included or excluded from the range.
- */
- public TrieRangeQuery(final String field, final Long min, final Long max,
- final boolean minInclusive, final boolean maxInclusive, final TrieUtils variant
- ) {
- super(new TrieRangeFilter(field,min,max,minInclusive,maxInclusive,variant));
- }
-
- /**
- * EXPERT: Return the number of terms visited during the last execution of the query.
- * This may be used for performance comparisons of different trie variants and their effectiveness.
- * When using this method be sure to query an one-segment index (optimized one) to get correct results.
- * This method is not thread safe, be sure to only call it when no query is running!
- * @throws IllegalStateException if query was not yet executed.
- */
- public int getLastNumberOfTerms() {
- return ((TrieRangeFilter) filter).getLastNumberOfTerms();
- }
-
- //@Override
- public String toString(final String field) {
- // return a more convenient representation of this query than ConstantScoreQuery does:
- return ((TrieRangeFilter) filter).toString(field)+ToStringUtils.boost(getBoost());
- }
-
- /**
- * Two instances are equal if they have the same trie-encoded range bounds, same field, same boost, and same variant.
- * If one of the instances uses an exclusive lower bound, it is equal to a range with inclusive bound,
- * when the inclusive lower bound is equal to the decremented exclusive lower bound.
- * The same applys for the upper bound in other direction.
- */
- //@Override
- public final boolean equals(final Object o) {
- if (!(o instanceof TrieRangeQuery)) return false;
- return super.equals(o);
- }
-
- //@Override
- public final int hashCode() {
- // make hashCode a little bit different:
- return super.hashCode()^0x1756fa55;
- }
-
-}
Index: contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (revision 742672)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (working copy)
@@ -17,385 +17,526 @@
* limitations under the License.
*/
-import java.util.Date;
-
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.SortField;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.ExtendedFieldCache;
/**
* This is a helper class to construct the trie-based index entries for numerical values.
- *
For more information on how the algorithm works, see the package description {@link org.apache.lucene.search.trie}. - * The format of how the numerical values are stored in index is documented here: - *
All numerical values are first converted to special unsigned longs by applying some bit-wise transformations. This means:
signed long is transformed to the unsigned form like so:signed longs are shifted, so that {@link Long#MIN_VALUE} is mapped to 0x0000000000000000,
- * {@link Long#MAX_VALUE} is mapped to 0xffffffffffffffff.doubles are converted by getting their IEEE 754 floating-point "double format" bit layout and then some bits
- * are swapped, to be able to compare the result as unsigned longs.For each variant (you can choose between {@link #VARIANT_8BIT}, {@link #VARIANT_4BIT}, and {@link #VARIANT_2BIT}),
- * the bitmap of this unsigned long is divided into parts of a number of bits (starting with the most-significant bits)
- * and each part converted to characters between {@link #TRIE_CODED_SYMBOL_MIN} and {@link #TRIE_CODED_SYMBOL_MAX}.
- * The resulting {@link String} is comparable like the corresponding unsigned long.
- *
To store the different precisions of the long values (from one character [only the most significant one] to the full encoded length),
- * each lower precision is prefixed by the length ({@link #TRIE_CODED_PADDING_START}+precision == 0x20+precision),
- * in an extra "helper" field with a suffixed field name (i.e. fieldname "numeric" => lower precision's name "numeric#trie").
- * The full long is not prefixed at all and indexed and stored according to the given flags in the original field name.
- * By this it is possible to get the correct enumeration of terms in correct precision
- * of the term list by just jumping to the correct fieldname and/or prefix. The full precision value may also be
- * stored in the document. Having the full precision value as term in a separate field with the original name,
- * sorting of query results against such fields is possible using the original field name.
+ * For more information on how the algorithm works, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
+ *
To quickly execute range queries in Apache Lucene, a range is divided recursively + * into multiple intervals for searching: The center of the range is searched only with + * the lowest possible precision in the trie, while the boundaries are matched + * more exactly. This reduces the number of terms dramatically. + *
This class generates terms to achive this: First the numerical integer values need to + * be converted to strings. For that integer values (32 bit or 64 bit) are made unsigned + * and the bits are converted to ASCII chars with each 7 bit. The resulting string is + * sortable like the original integer value. + *
To also index floating point numbers, this class supplies two methods to convert them + * to integer values by changing their bit layout: {@link #doubleToSortableLong}, + * {@link #floatToSortableInt}. You will have no precision loss by + * converting floating point numbers to integers and back (only that the integer form does + * is not usable). Other data types like dates can easily converted to longs or ints (e.g. + * date to long: {@link java.util.Date#getTime}). + *
To index the different precisions of the long values each encoded value is also reduced
+ * by zeroing bits from the right. Each value is also prefixed (in the first char) by the
+ * shift value (number of bits removed) used during encoding. This series of
+ * different precision values can be indexed into a Lucene {@link Document} using
+ * {@link #addIndexedFields(Document,String,String[])}. The default is to index the original
+ * precision in the supplied field name and the lower precisions in an additional helper field.
+ * Because of this, the full-precision field can also be sorted (using {@link #getLongSortField}
+ * or {@link #getIntSortField}).
+ *
The number of bits removed from the right for each trie entry is called
+ * precisionStep in this API. For comparing the different step values, see the
+ * {@linkplain org.apache.lucene.search.trie package description}.
*/
public final class TrieUtils {
- /** Instance of TrieUtils using a trie factor of 8 bit.
- * This is the recommended one (rather fast and storage optimized) */
- public static final TrieUtils VARIANT_8BIT=new TrieUtils(8);
+ private TrieUtils() {} // no instance!
- /** Instance of TrieUtils using a trie factor of 4 bit. */
- public static final TrieUtils VARIANT_4BIT=new TrieUtils(4);
-
- /** Instance of TrieUtils using a trie factor of 2 bit.
- * This may be good for some indexes, but it needs much storage space
- * and is not much faster than 8 bit in most cases. */
- public static final TrieUtils VARIANT_2BIT=new TrieUtils(2);
-
- /** Marker (PADDING) before lower-precision trie entries to signal the precision value. See class description! */
- public static final char TRIE_CODED_PADDING_START=(char)0x20;
-
- /** The "helper" field containing the lower precision terms is the original fieldname with this appended. */
+ /**
+ * The default "helper" field containing the lower precision terms is the original
+ * fieldname with this appended. This suffix is used in
+ * {@link #addIndexedFields(Document,String,String[])} and the corresponding c'tor
+ * of (Long|Int)TrieRangeFilter.
+ */
public static final String LOWER_PRECISION_FIELD_NAME_SUFFIX="#trie";
- /** Character used as lower end */
- public static final char TRIE_CODED_SYMBOL_MIN=(char)0x100;
+ /**
+ * Longs are stored at lower precision by shifting off lower bits. The shift count is
+ * stored as SHIFT_START_LONG+shift in the first character
+ */
+ public static final char SHIFT_START_LONG = (char)0x20;
/**
- * A parser instance for filling a {@link ExtendedFieldCache}, that parses trie encoded fields as longs,
- * auto detecting the trie encoding variant using the String length.
+ * Integers are stored at lower precision by shifting off lower bits. The shift count is
+ * stored as SHIFT_START_INT+shift in the first character
*/
- public static final ExtendedFieldCache.LongParser FIELD_CACHE_LONG_PARSER_AUTO=new ExtendedFieldCache.LongParser(){
- public final long parseLong(String val) {
- return trieCodedToLongAuto(val);
+ public static final char SHIFT_START_INT = (char)0x60;
+
+ /**
+ * A parser instance for filling a {@link ExtendedFieldCache}, that parses prefix encoded fields as longs.
+ */
+ public static final ExtendedFieldCache.LongParser FIELD_CACHE_LONG_PARSER=new ExtendedFieldCache.LongParser(){
+ public final long parseLong(final String val) {
+ return prefixCodedToLong(val);
}
};
/**
- * A parser instance for filling a {@link ExtendedFieldCache}, that parses trie encoded fields as doubles,
- * auto detecting the trie encoding variant using the String length.
+ * A parser instance for filling a {@link FieldCache}, that parses prefix encoded fields as ints.
*/
- public static final ExtendedFieldCache.DoubleParser FIELD_CACHE_DOUBLE_PARSER_AUTO=new ExtendedFieldCache.DoubleParser(){
- public final double parseDouble(String val) {
- return trieCodedToDoubleAuto(val);
+ public static final FieldCache.IntParser FIELD_CACHE_INT_PARSER=new FieldCache.IntParser(){
+ public final int parseInt(final String val) {
+ return prefixCodedToInt(val);
}
};
+
+ /**
+ * A parser instance for filling a {@link ExtendedFieldCache}, that parses prefix encoded fields as doubles.
+ * This uses {@link #sortableLongToDouble} to convert the encoded long to a double.
+ */
+ public static final ExtendedFieldCache.DoubleParser FIELD_CACHE_DOUBLE_PARSER=new ExtendedFieldCache.DoubleParser(){
+ public final double parseDouble(final String val) {
+ return sortableLongToDouble(prefixCodedToLong(val));
+ }
+ };
/**
- * Detects and returns the variant of a trie encoded string using the length.
- * @throws NumberFormatException if the length is not 8, 16, or 32 chars.
+ * A parser instance for filling a {@link FieldCache}, that parses prefix encoded fields as floats.
+ * This uses {@link #sortableIntToFloat} to convert the encoded int to a float.
*/
- public static final TrieUtils autoDetectVariant(final String s) {
- final int l=s.length();
- if (l==VARIANT_8BIT.TRIE_CODED_LENGTH) {
- return VARIANT_8BIT;
- } else if (l==VARIANT_4BIT.TRIE_CODED_LENGTH) {
- return VARIANT_4BIT;
- } else if (l==VARIANT_2BIT.TRIE_CODED_LENGTH) {
- return VARIANT_2BIT;
- } else {
- throw new NumberFormatException("Invalid trie encoded numerical value representation (incompatible length).");
+ public static final FieldCache.FloatParser FIELD_CACHE_FLOAT_PARSER=new FieldCache.FloatParser(){
+ public final float parseFloat(final String val) {
+ return sortableIntToFloat(prefixCodedToInt(val));
}
- }
+ };
/**
- * Converts a encoded String value back to a long,
- * auto detecting the trie encoding variant using the String length.
+ * This is a convenience method, that returns prefix coded bits of a long without
+ * reducing the precision. It can be used to store the full precision value as a
+ * stored field in index.
+ *
To decode, use {@link #prefixCodedToLong}.
*/
- public static final long trieCodedToLongAuto(final String s) {
- return autoDetectVariant(s).trieCodedToLong(s);
+ public static String longToPrefixCoded(final long val) {
+ return longToPrefixCoded(val, 0);
}
/**
- * Converts a encoded String value back to a double,
- * auto detecting the trie encoding variant using the String length.
+ * Expert: Returns prefix coded bits after reducing the precision by shift bits.
+ * This is method is used by {@link #trieCodeLong}.
*/
- public static final double trieCodedToDoubleAuto(final String s) {
- return autoDetectVariant(s).trieCodedToDouble(s);
+ public static String longToPrefixCoded(final long val, final int shift) {
+ if (shift>63 || shift<0)
+ throw new IllegalArgumentException("Illegal shift value, must be 0..63");
+ int nChars = (63-shift)/7 + 1;
+ final char[] arr = new char[nChars+1];
+ arr[0] = (char)(SHIFT_START_LONG + shift);
+ long sortableBits = val ^ 0x8000000000000000L;
+ sortableBits >>>= shift;
+ while (nChars>=1) {
+ // Store 7 bits per character for good efficiency when UTF-8 encoding.
+ // The whole number is right-justified so that lucene can prefix-encode
+ // the terms more efficiently.
+ arr[nChars--] = (char)(sortableBits & 0x7f);
+ sortableBits >>>= 7;
+ }
+ return new String(arr);
}
/**
- * Converts a encoded String value back to a Date,
- * auto detecting the trie encoding variant using the String length.
+ * This is a convenience method, that returns prefix coded bits of an int without
+ * reducing the precision. It can be used to store the full precision value as a
+ * stored field in index.
+ *
To decode, use {@link #prefixCodedToInt}.
*/
- public static final Date trieCodedToDateAuto(final String s) {
- return autoDetectVariant(s).trieCodedToDate(s);
+ public static String intToPrefixCoded(final int val) {
+ return intToPrefixCoded(val, 0);
}
/**
- * A factory method, that generates a {@link SortField} instance for sorting trie encoded values,
- * automatically detecting the trie encoding variant using the String length.
+ * Expert: Returns prefix coded bits after reducing the precision by To achieve this, use {@link #addIndexedFields(Document,String,String[])}.
+ */
+ public static String[] trieCodeLong(long val, int precisionStep) {
+ String[] arr = new String[63/precisionStep+1];
+ int idx = 0;
+ for (int shift=0; shift<64; shift+=precisionStep) {
+ arr[idx++] = longToPrefixCoded(val, shift);
}
- return Double.longBitsToDouble(l);
+ return arr;
}
- // Date's
-
- /** Converts a To achieve this, use {@link #addIndexedFields(Document,String,String[])}.
+ */
+ public static String[] trieCodeInt(int val, int precisionStep) {
+ String[] arr = new String[31/precisionStep+1];
+ int idx = 0;
+ for (int shift=0; shift<32; shift+=precisionStep) {
+ arr[idx++] = intToPrefixCoded(val, shift);
+ }
+ return arr;
}
- /** Converts a encoded This is the recommended variant to add trie fields to the index.
+ * By this it is possible to sort the field using a This method does not store the fields and saves no term frequency or norms
+ * (which are normally not needed for trie fields). If you want to additionally store
+ * the value, you can use the normal methods of {@link Document} to achive this, just specify
+ * Examples:
+ * This method does not store the fields and saves no term frequency or norms
+ * (which are normally not needed for trie fields). If you want to additionally store
+ * the value, you can use the normal methods of {@link Document} to achive this, just specify
+ * Examples:
+ * This method does not store the fields and saves no term frequency or norms
+ * (which are normally not needed for trie fields). If you want to additionally store
+ * the value, you can use the normal methods of {@link Document} to achive this, just specify
+ * To store the different precisions of the long values (from one byte [only the most significant one] to the full eight bytes),
- * each lower precision is prefixed by the length ({@link #TRIE_CODED_PADDING_START} This method is used by {@link LongTrieRangeFilter}.
*/
- public void addDoubleTrieCodedDocumentField(
- final Document ldoc, final String fieldname, final double val,
- final boolean index, final Field.Store store
+ public static void splitLongRange(final LongRangeBuilder builder,
+ final int precisionStep, final long minBound, final long maxBound
) {
- addConvertedTrieCodedDocumentField(ldoc, fieldname, doubleToTrieCoded(val), index, store);
+ splitRange(
+ builder, 64, precisionStep, minBound, maxBound,
+ 0 /* start with no shift */
+ );
}
-
+
/**
- * Stores a Date value in trie-form in document for indexing.
- * To store the different precisions of the long values (from one byte [only the most significant one] to the full eight bytes),
- * each lower precision is prefixed by the length ({@link #TRIE_CODED_PADDING_START} This method is used by {@link IntTrieRangeFilter}.
*/
- public void addDateTrieCodedDocumentField(
- final Document ldoc, final String fieldname,
- final Date val, final boolean index, final Field.Store store
+ public static void splitIntRange(final IntRangeBuilder builder,
+ final int precisionStep, final int minBound, final int maxBound
) {
- addConvertedTrieCodedDocumentField(ldoc, fieldname, dateToTrieCoded(val), index, store);
+ splitRange(
+ builder, 32, precisionStep, minBound, maxBound,
+ 0 /* start with no shift */
+ );
}
+
+ /** This helper does the splitting for both 32 and 64 bit. */
+ private static void splitRange(
+ final Object builder, final int valSize,
+ final int precisionStep, final long minBound, final long maxBound,
+ final int shift
+ ) {
+ // calculate new bounds for inner precision
+ final long diff = 1L << (shift+precisionStep),
+ mask = ((1L<shift bits.
+ * This is method is used by {@link #trieCodeInt}.
*/
- public static final SortField getSortFieldAuto(final String field) {
- return new SortField(field, FIELD_CACHE_LONG_PARSER_AUTO);
+ public static String intToPrefixCoded(final int val, final int shift) {
+ if (shift>31 || shift<0)
+ throw new IllegalArgumentException("Illegal shift value, must be 0..31");
+ int nChars = (31-shift)/7 + 1;
+ final char[] arr = new char[nChars+1];
+ arr[0] = (char)(SHIFT_START_INT + shift);
+ int sortableBits = val ^ 0x80000000;
+ sortableBits >>>= shift;
+ while (nChars>=1) {
+ // Store 7 bits per character for good efficiency when UTF-8 encoding.
+ // The whole number is right-justified so that lucene can prefix-encode
+ // the terms more efficiently.
+ arr[nChars--] = (char)(sortableBits & 0x7f);
+ sortableBits >>>= 7;
+ }
+ return new String(arr);
}
-
+
/**
- * A factory method, that generates a {@link SortField} instance for sorting trie encoded values,
- * automatically detecting the trie encoding variant using the String length.
+ * Returns a long from prefixCoded characters.
+ * Rightmost bits will be zero for lower precision codes.
+ * This method can be used to decode e.g. a stored field.
+ * @see #longToPrefixCoded(long)
*/
- public static final SortField getSortFieldAuto(final String field, boolean reverse) {
- return new SortField(field, FIELD_CACHE_LONG_PARSER_AUTO, reverse);
- }
-
- // TrieUtils instance's part
-
- private TrieUtils(int bits) {
- assert 64%bits == 0;
-
- // helper variable for conversion
- mask = (1L << bits) - 1L;
-
- // init global "constants"
- TRIE_BITS=bits;
- TRIE_CODED_LENGTH=64/TRIE_BITS;
- TRIE_CODED_SYMBOL_MAX=(char)(TRIE_CODED_SYMBOL_MIN+mask);
- TRIE_CODED_NUMERIC_MIN=longToTrieCoded(Long.MIN_VALUE);
- TRIE_CODED_NUMERIC_MAX=longToTrieCoded(Long.MAX_VALUE);
- }
-
- // internal conversion to/from strings
-
- private final String internalLongToTrieCoded(long l) {
- final char[] buf=new char[TRIE_CODED_LENGTH];
- for (int i=TRIE_CODED_LENGTH-1; i>=0; i--) {
- buf[i] = (char)( TRIE_CODED_SYMBOL_MIN + (l & mask) );
- l = l >>> TRIE_BITS;
+ public static long prefixCodedToLong(final String prefixCoded) {
+ final int len = prefixCoded.length();
+ final int shift = prefixCoded.charAt(0)-SHIFT_START_LONG;
+ if (shift>63 || shift<0)
+ throw new NumberFormatException("Invalid shift value in prefixCoded string (is encoded value really a LONG?)");
+ long sortableBits = 0L;
+ for (int i=1; idouble value to a sortable signed long.
+ * The value is converted by getting their IEEE 754 floating-point "double format"
+ * bit layout and then some bits are swapped, to be able to compare the result as long.
+ * By this the precision is not reduced, but the value can easily used as a long.
+ * @see #sortableLongToDouble
+ */
+ public static long doubleToSortableLong(double val) {
+ long f = Double.doubleToLongBits(val);
+ if (f<0) f ^= 0x7fffffffffffffffL;
+ return f;
+ }
- /** Converts a long value encoded to a String. */
- public String longToTrieCoded(final long l) {
- return internalLongToTrieCoded(l ^ 0x8000000000000000L);
+ /**
+ * Converts a sortable long back to a double.
+ * @see #doubleToSortableLong
+ */
+ public static double sortableLongToDouble(long val) {
+ if (val<0) val ^= 0x7fffffffffffffffL;
+ return Double.longBitsToDouble(val);
}
- /** Converts a encoded String value back to a long. */
- public long trieCodedToLong(final String s) {
- return internalTrieCodedToLong(s) ^ 0x8000000000000000L;
+ /**
+ * Converts a float value to a sortable signed int.
+ * The value is converted by getting their IEEE 754 floating-point "float format"
+ * bit layout and then some bits are swapped, to be able to compare the result as int.
+ * By this the precision is not reduced, but the value can easily used as an int.
+ * @see #sortableIntToFloat
+ */
+ public static int floatToSortableInt(float val) {
+ int f = Float.floatToIntBits(val);
+ if (f<0) f ^= 0x7fffffff;
+ return f;
}
- // Double's
+ /**
+ * Converts a sortable int back to a float.
+ * @see #floatToSortableInt
+ */
+ public static float sortableIntToFloat(int val) {
+ if (val<0) val ^= 0x7fffffff;
+ return Float.intBitsToFloat(val);
+ }
- /** Converts a double value encoded to a String. */
- public String doubleToTrieCoded(final double d) {
- long l=Double.doubleToLongBits(d);
- if ((l & 0x8000000000000000L) == 0L) {
- // >0
- l |= 0x8000000000000000L;
- } else {
- // <0
- l = ~l;
- }
- return internalLongToTrieCoded(l);
+ /** A factory method, that generates a {@link SortField} instance for sorting prefix encoded long values. */
+ public static SortField getLongSortField(final String field, boolean reverse) {
+ return new SortField(field, FIELD_CACHE_LONG_PARSER, reverse);
}
+
+ /** A factory method, that generates a {@link SortField} instance for sorting prefix encoded int values. */
+ public static SortField getIntSortField(final String field, boolean reverse) {
+ return new SortField(field, FIELD_CACHE_INT_PARSER, reverse);
+ }
- /** Converts a encoded String value back to a double. */
- public double trieCodedToDouble(final String s) {
- long l=internalTrieCodedToLong(s);
- if ((l & 0x8000000000000000L) != 0L) {
- // >0
- l &= 0x7fffffffffffffffL;
- } else {
- // <0
- l = ~l;
+ /**
+ * Returns a sequence of trie coded numbers suitable for {@link LongTrieRangeFilter}.
+ * Each successive string in the list has had it's precision reduced by precisionStep.
+ * For sorting, index the first full-precision value into a separate field and the
+ * remaining values into another field.
+ * Date value encoded to a String. */
- public String dateToTrieCoded(final Date d) {
- return longToTrieCoded(d.getTime());
+ /**
+ * Returns a sequence of trie coded numbers suitable for {@link IntTrieRangeFilter}.
+ * Each successive string in the list has had it's precision reduced by precisionStep.
+ * For sorting, index the first full-precision value into a separate field and the
+ * remaining values into another field.
+ * String value back to a Date. */
- public Date trieCodedToDate(final String s) {
- return new Date(trieCodedToLong(s));
+ /**
+ * Indexes the full precision value only in the main field (for sorting), and indexes all other
+ * lower precision values in field+LOWER_PRECISION_FIELD_NAME_SUFFIX.
+ * SortField instance
+ * returned by {@link #getLongSortField} or {@link #getIntSortField}.
+ * Field.Store.YES, Field.Index.NO and the same field name.
+ *
+ * addIndexedFields(doc, "mydouble", trieCodeLong(doubleToSortableLong(1.414d), 4));
+ * addIndexedFields(doc, "mylong", trieCodeLong(123456L, 4));
+ *
+ **/
+ public static void addIndexedFields(Document doc, String field, String[] trieCoded) {
+ addIndexedFields(doc, new String[]{field, field+LOWER_PRECISION_FIELD_NAME_SUFFIX}, trieCoded);
}
- // increment / decrement
-
- /** Increments an encoded String value by 1. Needed by {@link TrieRangeFilter}. */
- public String incrementTrieCoded(final String v) {
- final int l=v.length();
- final char[] buf=new char[l];
- boolean inc=true;
- for (int i=l-1; i>=0; i--) {
- int b=v.charAt(i)-TRIE_CODED_SYMBOL_MIN;
- if (inc) b++;
- if (inc=(b>(int)mask)) b=0;
- buf[i]=(char)(TRIE_CODED_SYMBOL_MIN+b);
- }
- return new String(buf);
+ /**
+ * Expert: Indexes the full precision value only in the main field (for sorting), and indexes all other
+ * lower precision values in the lowerPrecision field.
+ * If you do not specify the same field name for the main and lower precision one,
+ * it is possible to sort the field using a SortField instance
+ * returned by {@link #getLongSortField} or {@link #getIntSortField}.
+ * Field.Store.YES, Field.Index.NO and the same main field name.
+ *
+ * addIndexedFields(doc, "mydouble", "mydoubletrie", trieCodeLong(doubleToSortableLong(1.414d), 4));
+ * addIndexedFields(doc, "mylong", "mylongtrie", trieCodeLong(123456L, 4));
+ *
+ * @see #addIndexedFields(Document,String,String[])
+ **/
+ public static void addIndexedFields(Document doc, String field, String lowerPrecisionField, String[] trieCoded) {
+ addIndexedFields(doc, new String[]{field, lowerPrecisionField}, trieCoded);
}
- /** Decrements an encoded String value by 1. Needed by {@link TrieRangeFilter}. */
- public String decrementTrieCoded(final String v) {
- final int l=v.length();
- final char[] buf=new char[l];
- boolean dec=true;
- for (int i=l-1; i>=0; i--) {
- int b=v.charAt(i)-TRIE_CODED_SYMBOL_MIN;
- if (dec) b--;
- if (dec=(b<0)) b=(int)mask;
- buf[i]=(char)(TRIE_CODED_SYMBOL_MIN+b);
- }
- return new String(buf);
- }
-
- private void addConvertedTrieCodedDocumentField(
- final Document ldoc, final String fieldname, final String val,
- final boolean index, final Field.Store store
- ) {
- Field f=new Field(fieldname, val, store, index?Field.Index.NOT_ANALYZED_NO_NORMS:Field.Index.NO);
- if (index) {
+ /**
+ * Expert: Indexes a series of trie coded values into a lucene {@link Document}
+ * using the given field names.
+ * If the array of field names is shorter than the trie coded one, all trie coded
+ * values with higher index get the last field name.
+ * Field.Store.YES, Field.Index.NO and the same main field name.
+ **/
+ public static void addIndexedFields(Document doc, String[] fields, String[] trieCoded) {
+ for (int i=0; i+precision),
- * in an extra "helper" field with a name of fieldname+{@link #LOWER_PRECISION_FIELD_NAME_SUFFIX}
- * (i.e. fieldname "numeric" => lower precision's name "numeric#trie").
- * The full long is not prefixed at all and indexed and stored according to the given flags in the original field name.
- * If the field should not be searchable, set index to false. It is then only stored (for convenience).
- * Fields added to a document using this method can be queried by {@link TrieRangeQuery}.
+ * Expert: Splits a long range recursively.
+ * You may implement a builder that adds clauses to a
+ * {@link org.apache.lucene.search.BooleanQuery} for each call to its
+ * {@link IntRangeBuilder#addRange(String,String,int)}
+ * method.
+ * +precision),
- * in an extra "helper" field with a name of fieldname+{@link #LOWER_PRECISION_FIELD_NAME_SUFFIX}
- * (i.e. fieldname "numeric" => lower precision's name "numeric#trie").
- * The full long is not prefixed at all and indexed and stored according to the given flags in the original field name.
- * If the field should not be searchable, set index to false. It is then only stored (for convenience).
- * Fields added to a document using this method can be queried by {@link TrieRangeQuery}.
+ * Expert: Splits an int range recursively.
+ * You may implement a builder that adds clauses to a
+ * {@link org.apache.lucene.search.BooleanQuery} for each call to its
+ * {@link IntRangeBuilder#addRange(String,String,int)}
+ * method.
+ * +precision),
- * in an extra "helper" field with a name of fieldname+{@link #LOWER_PRECISION_FIELD_NAME_SUFFIX}
- * (i.e. fieldname "numeric" => lower precision's name "numeric#trie").
- * The full long is not prefixed at all and indexed and stored according to the given flags in the original field name.
- * If the field should not be searchable, set index to false. It is then only stored (for convenience).
- * Fields added to a document using this method can be queried by {@link TrieRangeQuery}.
- */
- public void addLongTrieCodedDocumentField(
- final Document ldoc, final String fieldname,
- final long val, final boolean index, final Field.Store store
- ) {
- addConvertedTrieCodedDocumentField(ldoc, fieldname, longToTrieCoded(val), index, store);
+ if (shift+precisionStep>=valSize || nextMinBound>=nextMaxBound) {
+ // we are in the lowest precision or the next precision is not available
+ addRange(builder, valSize, precisionStep, minBound, maxBound, shift);
+ } else {
+ // do we need a lower range?
+ if ((minBound & mask) != 0L) {
+ // add the lower range
+ addRange(builder, valSize, precisionStep, minBound, minBound | mask, shift);
+ } else {
+ // correct the next lower bound
+ nextMinBound = minBound & ~mask;
+ }
+ // do we need an upper range?
+ if ((maxBound & mask) != mask) {
+ // add the upper range
+ addRange(builder, valSize, precisionStep, maxBound & ~mask, maxBound, shift);
+ } else {
+ // correct the next upper bound
+ nextMaxBound = maxBound & ~mask;
+ }
+ // recurse down to next precision
+ splitRange(
+ builder, valSize, precisionStep,
+ nextMinBound, nextMaxBound,
+ shift+precisionStep
+ );
+ }
}
- /** A factory method, that generates a {@link SortField} instance for sorting trie encoded values. */
- public SortField getSortField(final String field) {
- return new SortField(field, FIELD_CACHE_LONG_PARSER);
+ /** Helper that delegates to correct range builder */
+ private static void addRange(
+ final Object builder, final int valSize,
+ final int precisionStep, final long minBound, final long maxBound,
+ final int shift
+ ) {
+ switch(valSize) {
+ case 64:
+ /*System.out.println(Long.toHexString((minBound^0x8000000000000000L) >>> shift)+".."+
+ Long.toHexString((maxBound^0x8000000000000000L) >>> shift));*/
+ ((LongRangeBuilder)builder).addRange(precisionStep, minBound, maxBound, shift);
+ break;
+ case 32:
+ /*System.out.println(Integer.toHexString((((int)minBound)^0x80000000) >>> shift)+".."+
+ Integer.toHexString((((int)maxBound)^0x80000000) >>> shift));*/
+ ((IntRangeBuilder)builder).addRange(precisionStep, (int)minBound, (int)maxBound, shift);
+ break;
+ default:
+ // Should not happen!
+ throw new IllegalArgumentException("valSize must be 32 or 64.");
+ }
}
+
+ /**
+ * Expert: Callback for {@link #splitLongRange}.
+ * You need to overwrite only one of the methods.
+ */
+ public static abstract class LongRangeBuilder {
+
+ /**
+ * Overwrite this method, if you like to receive the already prefix encoded range bounds.
+ * You can directly build classical range queries from them.
+ * The level gives the precision level (0 = highest precision) of the encoded values.
+ * This parameter could be used as an index to an array of fieldnames like the
+ * parameters to {@link #addIndexedFields(Document,String[],String[])} for specifying
+ * the field names for each precision:
+ *
+ * String field = fields[Math.min(fields.length-1, level)];
+ *
+ */
+ public void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Overwrite this method, if you like to receive the raw long range bounds.
+ * You can use this for e.g. debugging purposes (print out range bounds).
+ */
+ public void addRange(final int precisionStep, final long min, final long max, final int shift) {
+ addRange(longToPrefixCoded(min, shift), longToPrefixCoded(max, shift), shift/precisionStep);
+ }
- /** A factory method, that generates a {@link SortField} instance for sorting trie encoded values. */
- public SortField getSortField(final String field, boolean reverse) {
- return new SortField(field, FIELD_CACHE_LONG_PARSER, reverse);
}
- /** A parser instance for filling a {@link ExtendedFieldCache}, that parses trie encoded fields as longs. */
- public final ExtendedFieldCache.LongParser FIELD_CACHE_LONG_PARSER=new ExtendedFieldCache.LongParser(){
- public final long parseLong(String val) {
- return trieCodedToLong(val);
+ /**
+ * Expert: Callback for {@link #splitIntRange}.
+ * You need to overwrite only one of the methods.
+ */
+ public static abstract class IntRangeBuilder {
+
+ /**
+ * Overwrite this method, if you like to receive the already prefix encoded range bounds.
+ * You can directly build classical range queries from them.
+ * The level gives the precision level (0 = highest precision) of the encoded values.
+ * This parameter could be used as an index to an array of fieldnames like the
+ * parameters to {@link #addIndexedFields(Document,String[],String[])} for specifying
+ * the field names for each precision:
+ *
+ * String field = fields[Math.min(fields.length-1, level)];
+ *
+ */
+ public void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
+ throw new UnsupportedOperationException();
}
- };
-
- /** A parser instance for filling a {@link ExtendedFieldCache}, that parses trie encoded fields as doubles. */
- public final ExtendedFieldCache.DoubleParser FIELD_CACHE_DOUBLE_PARSER=new ExtendedFieldCache.DoubleParser(){
- public final double parseDouble(String val) {
- return trieCodedToDouble(val);
+
+ /**
+ * Overwrite this method, if you like to receive the raw int range bounds.
+ * You can use this for e.g. debugging purposes (print out range bounds).
+ */
+ public void addRange(final int precisionStep, final int min, final int max, final int shift) {
+ addRange(intToPrefixCoded(min, shift), intToPrefixCoded(max, shift), shift/precisionStep);
}
- };
- private final long mask;
+ }
- /** Number of bits used in this trie variant (2, 4, or 8) */
- public final int TRIE_BITS;
-
- /** Length (in chars) of an encoded value (8, 16, or 32 chars) */
- public final int TRIE_CODED_LENGTH;
-
- /** Character used as upper end (depends on trie bits, its {@link #TRIE_CODED_SYMBOL_MIN}+2^{@link #TRIE_BITS}-1) */
- public final char TRIE_CODED_SYMBOL_MAX;
-
- /** minimum encoded value of a numerical index entry: {@link Long#MIN_VALUE} */
- public final String TRIE_CODED_NUMERIC_MIN;
-
- /** maximum encoded value of a numerical index entry: {@link Long#MAX_VALUE} */
- public final String TRIE_CODED_NUMERIC_MAX;
-
}
Index: contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java
===================================================================
--- contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java (revision 0)
+++ contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java (revision 0)
@@ -0,0 +1,285 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Random;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriter.MaxFieldLength;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.search.RangeQuery;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestLongTrieRangeFilter extends LuceneTestCase
+{
+ private static final long distance=66666;
+
+ private static final RAMDirectory directory;
+ private static final IndexSearcher searcher;
+ static {
+ try {
+ directory = new RAMDirectory();
+ IndexWriter writer = new IndexWriter(directory, new WhitespaceAnalyzer(),
+ true, MaxFieldLength.UNLIMITED);
+
+ // Add a series of 10000 docs with increasing long values
+ for (long l=0L; l<10000L; l++) {
+ Document doc=new Document();
+ // add fields, that have a distance to test general functionality
+ TrieUtils.addIndexedFields(doc,"field8", TrieUtils.trieCodeLong(distance*l, 8));
+ doc.add(new Field("field8", TrieUtils.longToPrefixCoded(distance*l), Field.Store.YES, Field.Index.NO));
+ TrieUtils.addIndexedFields(doc,"field4", TrieUtils.trieCodeLong(distance*l, 4));
+ doc.add(new Field("field4", TrieUtils.longToPrefixCoded(distance*l), Field.Store.YES, Field.Index.NO));
+ TrieUtils.addIndexedFields(doc,"field2", TrieUtils.trieCodeLong(distance*l, 2));
+ doc.add(new Field("field2", TrieUtils.longToPrefixCoded(distance*l), Field.Store.YES, Field.Index.NO));
+ // add ascending fields with a distance of 1 to test the correct splitting of range and inclusive/exclusive
+ TrieUtils.addIndexedFields(doc,"ascfield8", TrieUtils.trieCodeLong(l, 8));
+ TrieUtils.addIndexedFields(doc,"ascfield4", TrieUtils.trieCodeLong(l, 4));
+ TrieUtils.addIndexedFields(doc,"ascfield2", TrieUtils.trieCodeLong(l, 2));
+ writer.addDocument(doc);
+ }
+
+ writer.optimize();
+ writer.close();
+ searcher=new IndexSearcher(directory);
+ } catch (Exception e) {
+ throw new Error(e);
+ }
+ }
+
+ private void testRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long lower=96666L, upper=lower + count*distance + 1234L;
+ LongTrieRangeFilter f=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ TopDocs topDocs = searcher.search(f.asQuery(), null, 10000, Sort.INDEXORDER);
+ System.out.println("Found "+f.getLastNumberOfTerms()+" distinct terms in range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score docs must match "+count+" docs, found "+sd.length+" docs", sd.length, count );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc should be "+(2*distance), TrieUtils.prefixCodedToLong(doc.get(field)), 2*distance );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc should be "+((1+count)*distance), TrieUtils.prefixCodedToLong(doc.get(field)), (1+count)*distance );
+ }
+
+ public void testRange_8bit() throws Exception {
+ testRange(8);
+ }
+
+ public void testRange_4bit() throws Exception {
+ testRange(4);
+ }
+
+ public void testRange_2bit() throws Exception {
+ testRange(2);
+ }
+
+ private void testLeftOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long upper=(count-1)*distance + 1234L;
+ LongTrieRangeFilter f=new LongTrieRangeFilter(field, precisionStep, null, new Long(upper), true, true);
+ TopDocs topDocs = searcher.search(f.asQuery(), null, 10000, Sort.INDEXORDER);
+ System.out.println("Found "+f.getLastNumberOfTerms()+" distinct terms in left open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score docs must match "+count+" docs, found "+sd.length+" docs", sd.length, count );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc should be 0", TrieUtils.prefixCodedToLong(doc.get(field)), 0L );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc should be "+((count-1)*distance), TrieUtils.prefixCodedToLong(doc.get(field)), (count-1)*distance );
+ }
+
+ public void testLeftOpenRange_8bit() throws Exception {
+ testLeftOpenRange(8);
+ }
+
+ public void testLeftOpenRange_4bit() throws Exception {
+ testLeftOpenRange(4);
+ }
+
+ public void testLeftOpenRange_2bit() throws Exception {
+ testLeftOpenRange(2);
+ }
+
+ private void testRightOpenRange(int precisionStep) throws Exception {
+ String field="field"+precisionStep;
+ int count=3000;
+ long lower=(count-1)*distance + 1234L;
+ LongTrieRangeFilter f=new LongTrieRangeFilter(field, precisionStep, new Long(lower), null, true, true);
+ TopDocs topDocs = searcher.search(f.asQuery(), null, 10000, Sort.INDEXORDER);
+ System.out.println("Found "+f.getLastNumberOfTerms()+" distinct terms in right open range for field '"+field+"'.");
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ assertEquals("Score docs must match "+(10000-count)+" docs, found "+sd.length+" docs", sd.length, 10000-count );
+ Document doc=searcher.doc(sd[0].doc);
+ assertEquals("First doc should be "+(count*distance), TrieUtils.prefixCodedToLong(doc.get(field)), count*distance );
+ doc=searcher.doc(sd[sd.length-1].doc);
+ assertEquals("Last doc should be "+9999L*distance, TrieUtils.prefixCodedToLong(doc.get(field)), 9999L*distance );
+ }
+
+ public void testRightOpenRange_8bit() throws Exception {
+ testRightOpenRange(8);
+ }
+
+ public void testRightOpenRange_4bit() throws Exception {
+ testRightOpenRange(4);
+ }
+
+ public void testRightOpenRange_2bit() throws Exception {
+ testRightOpenRange(2);
+ }
+
+ private void testRandomTrieAndClassicRangeQuery(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ // 50 random tests, the tests may also return 0 results, if min>max, but this is ok
+ for (int i=0; i<50; i++) {
+ long lower=(long)(rnd.nextDouble()*10000L*distance);
+ long upper=(long)(rnd.nextDouble()*10000L*distance);
+ // test inclusive range
+ Query tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true).asQuery();
+ RangeQuery cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), true, true);
+ cq.setConstantScoreRewrite(true);
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ TopDocs cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", tTopDocs.totalHits, cTopDocs.totalHits );
+ // test exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, false).asQuery();
+ cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), false, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", tTopDocs.totalHits, cTopDocs.totalHits );
+ // test left exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, true).asQuery();
+ cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), false, true);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", tTopDocs.totalHits, cTopDocs.totalHits );
+ // test right exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, false).asQuery();
+ cq=new RangeQuery(field, TrieUtils.longToPrefixCoded(lower), TrieUtils.longToPrefixCoded(upper), true, false);
+ cq.setConstantScoreRewrite(true);
+ tTopDocs = searcher.search(tq, 1);
+ cTopDocs = searcher.search(cq, 1);
+ assertEquals("Returned count for LongTrieRangeFilter and RangeQuery must be equal", tTopDocs.totalHits, cTopDocs.totalHits );
+ }
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_8bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(8);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_4bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(4);
+ }
+
+ public void testRandomTrieAndClassicRangeQuery_2bit() throws Exception {
+ testRandomTrieAndClassicRangeQuery(2);
+ }
+
+ private void testRangeSplit(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="ascfield"+precisionStep;
+ // 50 random tests
+ for (int i=0; i<50; i++) {
+ long lower=(long)(rnd.nextDouble()*10000L);
+ long upper=(long)(rnd.nextDouble()*10000L);
+ if (lower>upper) {
+ long a=lower; lower=upper; upper=a;
+ }
+ // test inclusive range
+ Query tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true).asQuery();
+ TopDocs tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to inclusive range length", tTopDocs.totalHits, upper-lower+1 );
+ // test exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, false).asQuery();
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to exclusive range length", tTopDocs.totalHits, Math.max(upper-lower-1, 0) );
+ // test left exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), false, true).asQuery();
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", tTopDocs.totalHits, upper-lower );
+ // test right exclusive range
+ tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, false).asQuery();
+ tTopDocs = searcher.search(tq, 1);
+ assertEquals("Returned count of range query must be equal to half exclusive range length", tTopDocs.totalHits, upper-lower );
+ }
+ }
+
+ public void testRangeSplit_8bit() throws Exception {
+ testRangeSplit(8);
+ }
+
+ public void testRangeSplit_4bit() throws Exception {
+ testRangeSplit(4);
+ }
+
+ public void testRangeSplit_2bit() throws Exception {
+ testRangeSplit(2);
+ }
+
+ private void testSorting(int precisionStep) throws Exception {
+ final Random rnd=newRandom();
+ String field="field"+precisionStep;
+ // 10 random tests, the index order is ascending,
+ // so using a reverse sort field should retun descending documents
+ for (int i=0; i<10; i++) {
+ long lower=(long)(rnd.nextDouble()*10000L*distance);
+ long upper=(long)(rnd.nextDouble()*10000L*distance);
+ if (lower>upper) {
+ long a=lower; lower=upper; upper=a;
+ }
+ LongTrieRangeFilter tq=new LongTrieRangeFilter(field, precisionStep, new Long(lower), new Long(upper), true, true);
+ TopDocs topDocs = searcher.search(tq.asQuery(), null, 10000, new Sort(TrieUtils.getLongSortField(field, true)));
+ if (topDocs.totalHits==0) continue;
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertNotNull(sd);
+ long last=TrieUtils.prefixCodedToLong(searcher.doc(sd[0].doc).get(field));
+ for (int j=1; j