Values indexed by this stream can be loaded into the {@link FieldCache}
* and can be sorted (use {@link SortField}{@code .TYPE} to specify the correct
- * type; {@link SortField#AUTO} does not work with this type of field)
+ * type; {@link SortField#AUTO} does not work with this type of field).
+ * Values solely used for sorting can be indexed using a precisionStep
+ * of 64 (long, double) or 32 (int, float),
+ * because this step only produces one value token with highest precision.
*
*
NOTE: This API is experimental and
* might change in incompatible ways in the next release.
@@ -86,22 +89,26 @@
*/
public final class NumericTokenStream extends TokenStream {
- /** The full precision 64 bit token gets this token type assigned. */
- public static final String TOKEN_TYPE_FULL_PREC_64 = "fullPrecNumeric64";
+ /** The full precision token gets this token type assigned. */
+ public static final String TOKEN_TYPE_FULL_PREC = "fullPrecNumeric";
- /** The lower precision 64 bit tokens gets this token type assigned. */
- public static final String TOKEN_TYPE_LOWER_PREC_64 = "lowerPrecNumeric64";
+ /** The lower precision tokens gets this token type assigned. */
+ public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
- /** The full precision 32 bit token gets this token type assigned. */
- public static final String TOKEN_TYPE_FULL_PREC_32 = "fullPrecNumeric32";
-
- /** The lower precision 32 bit tokens gets this token type assigned. */
- public static final String TOKEN_TYPE_LOWER_PREC_32 = "lowerPrecNumeric32";
-
/**
- * Creates a token stream for numeric values. The stream is not yet initialized,
+ * Creates a token stream for numeric values using the default precisionStep
+ * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The stream is not yet initialized,
* before using set a value using the various set???Value() methods.
*/
+ public NumericTokenStream() {
+ this(NumericUtils.PRECISION_STEP_DEFAULT);
+ }
+
+ /**
+ * Creates a token stream for numeric values with the specified
+ * precisionStep. The stream is not yet initialized,
+ * before using set a value using the various set???Value() methods.
+ */
public NumericTokenStream(final int precisionStep) {
this.precisionStep = precisionStep;
termAtt = (TermAttribute) addAttribute(TermAttribute.class);
@@ -180,15 +187,13 @@
final char[] buffer;
switch (valSize) {
case 64:
- buffer = termAtt.resizeTermBuffer(NumericUtils.LONG_BUF_SIZE);
+ buffer = termAtt.resizeTermBuffer(NumericUtils.BUF_SIZE_LONG);
termAtt.setTermLength(NumericUtils.longToPrefixCoded(value, shift, buffer));
- typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC_64 : TOKEN_TYPE_LOWER_PREC_64);
break;
case 32:
- buffer = termAtt.resizeTermBuffer(NumericUtils.INT_BUF_SIZE);
+ buffer = termAtt.resizeTermBuffer(NumericUtils.BUF_SIZE_INT);
termAtt.setTermLength(NumericUtils.intToPrefixCoded((int) value, shift, buffer));
- typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC_32 : TOKEN_TYPE_LOWER_PREC_32);
break;
default:
@@ -196,6 +201,7 @@
throw new IllegalArgumentException("valSize must be 32 or 64");
}
+ typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
shift += precisionStep;
return true;
@@ -215,15 +221,13 @@
final char[] buffer;
switch (valSize) {
case 64:
- buffer = reusableToken.resizeTermBuffer(NumericUtils.LONG_BUF_SIZE);
+ buffer = reusableToken.resizeTermBuffer(NumericUtils.BUF_SIZE_LONG);
reusableToken.setTermLength(NumericUtils.longToPrefixCoded(value, shift, buffer));
- reusableToken.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC_64 : TOKEN_TYPE_LOWER_PREC_64);
break;
case 32:
- buffer = reusableToken.resizeTermBuffer(NumericUtils.INT_BUF_SIZE);
+ buffer = reusableToken.resizeTermBuffer(NumericUtils.BUF_SIZE_INT);
reusableToken.setTermLength(NumericUtils.intToPrefixCoded((int) value, shift, buffer));
- reusableToken.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC_32 : TOKEN_TYPE_LOWER_PREC_32);
break;
default:
@@ -231,6 +235,7 @@
throw new IllegalArgumentException("valSize must be 32 or 64");
}
+ reusableToken.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
reusableToken.setPositionIncrement((shift == 0) ? 1 : 0);
shift += precisionStep;
return reusableToken;
Index: src/java/org/apache/lucene/document/NumericField.java
===================================================================
--- src/java/org/apache/lucene/document/NumericField.java (revision 793596)
+++ src/java/org/apache/lucene/document/NumericField.java (working copy)
@@ -21,6 +21,7 @@
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.NumericTokenStream;
+import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.search.NumericRangeQuery; // javadocs
import org.apache.lucene.search.NumericRangeFilter; // javadocs
import org.apache.lucene.search.SortField; // javadocs
@@ -72,7 +73,10 @@
*
*
Values indexed by this field can be loaded into the {@link FieldCache}
* and can be sorted (use {@link SortField}{@code .TYPE} to specify the correct
- * type; {@link SortField#AUTO} does not work with this type of field)
+ * type; {@link SortField#AUTO} does not work with this type of field).
+ * Values solely used for sorting can be indexed using a precisionStep
+ * of 64 (long, double) or 32 (int, float),
+ * because this step only produces one value token with highest precision.
*
*
NOTE: This API is experimental and
* might change in incompatible ways in the next release.
@@ -84,11 +88,38 @@
private final NumericTokenStream tokenStream;
/**
- * Creates a field for numeric values. The instance is not yet initialized with
+ * Creates a field for numeric values using the default precisionStep
+ * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The instance is not yet initialized with
* a numeric value, before indexing a document containing this field,
* set a value using the various set???Value() methods.
* This constrcutor creates an indexed, but not stored field.
* @param name the field name
+ */
+ public NumericField(String name) {
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, Field.Store.NO, true);
+ }
+
+ /**
+ * Creates a field for numeric values using the default precisionStep
+ * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The instance is not yet initialized with
+ * a numeric value, before indexing a document containing this field,
+ * set a value using the various set???Value() methods.
+ * @param name the field name
+ * @param store if the field should be stored in plain text form
+ * (according to toString(value) of the used data type)
+ * @param index if the field should be indexed using {@link NumericTokenStream}
+ */
+ public NumericField(String name, Field.Store store, boolean index) {
+ this(name, NumericUtils.PRECISION_STEP_DEFAULT, store, index);
+ }
+
+ /**
+ * Creates a field for numeric values with the specified
+ * precisionStep. The instance is not yet initialized with
+ * a numeric value, before indexing a document containing this field,
+ * set a value using the various set???Value() methods.
+ * This constrcutor creates an indexed, but not stored field.
+ * @param name the field name
* @param precisionStep the used precision step
*/
public NumericField(String name, int precisionStep) {
@@ -96,7 +127,8 @@
}
/**
- * Creates a field for numeric values. The instance is not yet initialized with
+ * Creates a field for numeric values with the specified
+ * precisionStep. The instance is not yet initialized with
* a numeric value, before indexing a document containing this field,
* set a value using the various set???Value() methods.
* @param name the field name
Index: src/java/org/apache/lucene/search/NumericRangeFilter.java
===================================================================
--- src/java/org/apache/lucene/search/NumericRangeFilter.java (revision 793596)
+++ src/java/org/apache/lucene/search/NumericRangeFilter.java (working copy)
@@ -19,6 +19,7 @@
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
import org.apache.lucene.document.NumericField; // for javadocs
+import org.apache.lucene.util.NumericUtils; // for javadocs
/**
* Implementation of a {@link Filter} that implements trie-based range filtering
@@ -65,6 +66,21 @@
}
/**
+ * Factory that creates a NumericRangeFilter, that queries a long
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeFilter newLongRange(final String field,
+ Long min, Long max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeFilter(
+ NumericRangeQuery.newLongRange(field, min, max, minInclusive, maxInclusive)
+ );
+ }
+
+ /**
* Factory that creates a NumericRangeFilter, that filters a int
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -80,6 +96,21 @@
}
/**
+ * Factory that creates a NumericRangeFilter, that queries a int
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeFilter newIntRange(final String field,
+ Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeFilter(
+ NumericRangeQuery.newIntRange(field, min, max, minInclusive, maxInclusive)
+ );
+ }
+
+ /**
* Factory that creates a NumericRangeFilter, that filters a double
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -95,6 +126,21 @@
}
/**
+ * Factory that creates a NumericRangeFilter, that queries a double
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeFilter newDoubleRange(final String field,
+ Double min, Double max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeFilter(
+ NumericRangeQuery.newDoubleRange(field, min, max, minInclusive, maxInclusive)
+ );
+ }
+
+ /**
* Factory that creates a NumericRangeFilter, that filters a float
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -109,6 +155,21 @@
);
}
+ /**
+ * Factory that creates a NumericRangeFilter, that queries a float
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeFilter newFloatRange(final String field,
+ Float min, Float max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeFilter(
+ NumericRangeQuery.newFloatRange(field, min, max, minInclusive, maxInclusive)
+ );
+ }
+
/** Returns the field name for this filter */
public String getField() { return ((NumericRangeQuery)query).getField(); }
Index: src/java/org/apache/lucene/search/NumericRangeQuery.java
===================================================================
--- src/java/org/apache/lucene/search/NumericRangeQuery.java (revision 793596)
+++ src/java/org/apache/lucene/search/NumericRangeQuery.java (working copy)
@@ -38,8 +38,9 @@
* An important setting is the precisionStep, which specifies,
* how many different precisions per numeric value are indexed to speed up range queries.
* Lower values create more terms but speed up search, higher values create less terms, but
- * slow down search. Suitable values are 2, 4, or 8. A good starting point to test is 4.
- * For code examples see {@link NumericField}.
+ * slow down search. Suitable values are between 1 and 8. A good starting point to test is 4,
+ * which is the default value for all Numeric* classes. For a discussion about ideal
+ * values, see below. Indexing code examples can be found in {@link NumericField}.
*
*
This class has no constructor, you can create queries depending on the data type
@@ -51,6 +52,8 @@
* new Float(0.3f), new Float(0.10f),
* true, true);
*
+ * The used precisionStep must be compatible
+ * to the one used during indexing (see below). The default is also 4.
*
*
Good values for precisionStep are depending on usage and data type:
+ *
precisionStep is given.
+ * precisionSteps.
+ * This dramatically improves the performance of Apache Lucene with range queries, which * are no longer dependent on the index size and the number of distinct values because there is * an upper limit unrelated to either of these properties.
* *Comparisions of the different types of RangeQueries on an index with about 500,000 docs showed - * that the old {@link RangeQuery} (with raised {@link BooleanQuery} clause count) took about 30-40 - * secs to complete, {@link ConstantScoreRangeQuery} took 5 secs and executing - * this class took <100ms to complete (on an Opteron64 machine, Java 1.5, 8 bit precision step). - * This query type was developed for a geographic portal, where the performance for + * that {@link TermRangeQuery} in boolean rewrite mode (with raised {@link BooleanQuery} clause count) + * took about 30-40 secs to complete, {@link TermRangeQuery} in constant score rewrite mode took 5 secs + * and executing this class took <100ms to complete (on an Opteron64 machine, Java 1.5, 8 bit + * precision step). This query type was developed for a geographic portal, where the performance for * e.g. bounding boxes or exact date/time stamps is important.
* - *The query is in {@linkplain #setConstantScoreRewrite constant score mode} per default. + *
The query defaults to {@linkplain #setConstantScoreRewrite constant score rewrite mode}.
* With precision steps of ≤4, this query can be run in conventional {@link BooleanQuery}
* rewrite mode without changing the max clause count.
*
@@ -153,6 +168,19 @@
}
/**
+ * Factory that creates a NumericRangeQuery, that queries a long
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeQuery newLongRange(final String field,
+ Long min, Long max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeQuery(field, NumericUtils.PRECISION_STEP_DEFAULT, 64, min, max, minInclusive, maxInclusive);
+ }
+
+ /**
* Factory that creates a NumericRangeQuery, that queries a int
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -166,6 +194,19 @@
}
/**
+ * Factory that creates a NumericRangeQuery, that queries a int
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeQuery newIntRange(final String field,
+ Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeQuery(field, NumericUtils.PRECISION_STEP_DEFAULT, 32, min, max, minInclusive, maxInclusive);
+ }
+
+ /**
* Factory that creates a NumericRangeQuery, that queries a double
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -179,6 +220,19 @@
}
/**
+ * Factory that creates a NumericRangeQuery, that queries a double
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeQuery newDoubleRange(final String field,
+ Double min, Double max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeQuery(field, NumericUtils.PRECISION_STEP_DEFAULT, 64, min, max, minInclusive, maxInclusive);
+ }
+
+ /**
* Factory that creates a NumericRangeQuery, that queries a float
* range using the given precisionStep.
* You can have half-open ranges (which are in fact </≤ or >/≥ queries)
@@ -191,6 +245,19 @@
return new NumericRangeQuery(field, precisionStep, 32, min, max, minInclusive, maxInclusive);
}
+ /**
+ * Factory that creates a NumericRangeQuery, that queries a float
+ * range using the default precisionStep {@link NumericUtils#PRECISION_STEP_DEFAULT} (4).
+ * You can have half-open ranges (which are in fact </≤ or >/≥ queries)
+ * by setting the min or max value to null. By setting inclusive to false, it will
+ * match all documents excluding the bounds, with inclusive on, the boundaries are hits, too.
+ */
+ public static NumericRangeQuery newFloatRange(final String field,
+ Float min, Float max, final boolean minInclusive, final boolean maxInclusive
+ ) {
+ return new NumericRangeQuery(field, NumericUtils.PRECISION_STEP_DEFAULT, 32, min, max, minInclusive, maxInclusive);
+ }
+
//@Override
protected FilteredTermEnum getEnum(final IndexReader reader) throws IOException {
return new NumericRangeTermEnum(reader);
Index: src/java/org/apache/lucene/util/NumericUtils.java
===================================================================
--- src/java/org/apache/lucene/util/NumericUtils.java (revision 793596)
+++ src/java/org/apache/lucene/util/NumericUtils.java (working copy)
@@ -18,6 +18,7 @@
*/
import org.apache.lucene.analysis.NumericTokenStream; // for javadocs
+import org.apache.lucene.document.NumericField; // for javadocs
import org.apache.lucene.search.NumericRangeQuery; // for javadocs
import org.apache.lucene.search.NumericRangeFilter; // for javadocs
@@ -62,9 +63,15 @@
public final class NumericUtils {
private NumericUtils() {} // no instance!
-
+
/**
- * Longs are stored at lower precision by shifting off lower bits. The shift count is
+ * The default precision step used by {@link NumericField}, {@link NumericTokenStream},
+ * {@link NumericRangeQuery}, and {@link NumericRangeFilter} as default
+ */
+ public static final int PRECISION_STEP_DEFAULT = 4;
+
+ /**
+ * Expert: Longs are stored at lower precision by shifting off lower bits. The shift count is
* stored as SHIFT_START_LONG+shift in the first character
*/
public static final char SHIFT_START_LONG = (char)0x20;
@@ -74,10 +81,10 @@
* for encoding long values.
* @see #longToPrefixCoded(long,int,char[])
*/
- public static final int LONG_BUF_SIZE = 63/7 + 2;
+ public static final int BUF_SIZE_LONG = 63/7 + 2;
/**
- * Integers are stored at lower precision by shifting off lower bits. The shift count is
+ * Expert: Integers are stored at lower precision by shifting off lower bits. The shift count is
* stored as SHIFT_START_INT+shift in the first character
*/
public static final char SHIFT_START_INT = (char)0x60;
@@ -87,14 +94,14 @@
* for encoding int values.
* @see #intToPrefixCoded(int,int,char[])
*/
- public static final int INT_BUF_SIZE = 31/7 + 2;
+ public static final int BUF_SIZE_INT = 31/7 + 2;
/**
* Expert: Returns prefix coded bits after reducing the precision by shift bits.
* This is method is used by {@link NumericTokenStream}.
* @param val the numeric value
* @param shift how many bits to strip from the right
- * @param buffer that will contain the encoded chars, must be at least of {@link #LONG_BUF_SIZE}
+ * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_LONG}
* length
* @return number of chars written to buffer
*/
@@ -122,7 +129,7 @@
* @param shift how many bits to strip from the right
*/
public static String longToPrefixCoded(final long val, final int shift) {
- final char[] buffer = new char[LONG_BUF_SIZE];
+ final char[] buffer = new char[BUF_SIZE_LONG];
final int len = longToPrefixCoded(val, shift, buffer);
return new String(buffer, 0, len);
}
@@ -142,7 +149,7 @@
* This is method is used by {@link NumericTokenStream}.
* @param val the numeric value
* @param shift how many bits to strip from the right
- * @param buffer that will contain the encoded chars, must be at least of {@link #INT_BUF_SIZE}
+ * @param buffer that will contain the encoded chars, must be at least of {@link #BUF_SIZE_INT}
* length
* @return number of chars written to buffer
*/
@@ -170,7 +177,7 @@
* @param shift how many bits to strip from the right
*/
public static String intToPrefixCoded(final int val, final int shift) {
- final char[] buffer = new char[INT_BUF_SIZE];
+ final char[] buffer = new char[BUF_SIZE_INT];
final int len = intToPrefixCoded(val, shift, buffer);
return new String(buffer, 0, len);
}
Index: src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (revision 793596)
+++ src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (working copy)
@@ -20,6 +20,7 @@
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
public class TestNumericTokenStream extends LuceneTestCase {
@@ -32,9 +33,11 @@
stream.setUseNewAPI(true);
// use getAttribute to test if attributes really exist, if not an IAE will be throwed
final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
+ final TypeAttribute typeAtt = (TypeAttribute) stream.getAttribute(TypeAttribute.class);
for (int shift=0; shift<64; shift+=precisionStep) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Term is correctly encoded", NumericUtils.longToPrefixCoded(lvalue, shift), termAtt.term());
+ assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("No more tokens available", stream.incrementToken());
}
@@ -46,6 +49,7 @@
for (int shift=0; shift<64; shift+=precisionStep) {
assertNotNull("New token is available", tok=stream.next(tok));
assertEquals("Term is correctly encoded", NumericUtils.longToPrefixCoded(lvalue, shift), tok.term());
+ assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, tok.type());
}
assertNull("No more tokens available", stream.next(tok));
}
@@ -55,9 +59,11 @@
stream.setUseNewAPI(true);
// use getAttribute to test if attributes really exist, if not an IAE will be throwed
final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
+ final TypeAttribute typeAtt = (TypeAttribute) stream.getAttribute(TypeAttribute.class);
for (int shift=0; shift<32; shift+=precisionStep) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Term is correctly encoded", NumericUtils.intToPrefixCoded(ivalue, shift), termAtt.term());
+ assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("No more tokens available", stream.incrementToken());
}
@@ -69,6 +75,7 @@
for (int shift=0; shift<32; shift+=precisionStep) {
assertNotNull("New token is available", tok=stream.next(tok));
assertEquals("Term is correctly encoded", NumericUtils.intToPrefixCoded(ivalue, shift), tok.term());
+ assertEquals("Type correct", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, tok.type());
}
assertNull("No more tokens available", stream.next(tok));
}