precisionStep must me equal or a multiple of the precisionStep
- * used for indexing the values.
- * You can leave the bounds open, by supplying null for min and/or
- * max. Inclusive/exclusive bounds can also be supplied.
- * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
- * This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
- */
- public IntTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
- final Integer min, final Integer max, final boolean minInclusive, final boolean maxInclusive
- ) {
- this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
- }
/**
- * Expert: A trie filter for matching trie coded values
- * using the given field names. If the array of field names is shorter than the
- * trieCoded one, all trieCoded values with higher index get the last field name.
- * precisionStep must me equal or a multiple of the precisionStep
- * used for indexing the values.
- * You can leave the bounds open, by supplying null for min and/or
- * max. Inclusive/exclusive bounds can also be supplied.
- * To query float values use the converter {@link TrieUtils#floatToSortableInt}.
- *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}.
- */
- public IntTrieRangeFilter(final String[] fields, final int precisionStep,
- Integer min, Integer max, final boolean minInclusive, final boolean maxInclusive
- ) {
- super(fields, precisionStep, min, max, minInclusive, maxInclusive);
- }
-
- /**
* Returns a DocIdSet that provides the documents which should be permitted or prohibited in search results.
*/
//@Override
@@ -116,7 +80,6 @@
try {
fillBits(
reader, bits, termDocs,
- fields[Math.min(fields.length-1, level)],
minPrefixCoded, maxPrefixCoded
);
} catch (IOException ioe) {
Index: contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java (revision 0)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/IntTrieTokenStream.java (revision 0)
@@ -0,0 +1,81 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public class IntTrieTokenStream extends TokenStream {
+
+ public static final String TOKEN_TYPE_FULLPREC = "fullPrecTrieInt";
+ public static final String TOKEN_TYPE_LOWERPREC = "lowerPrecTrieInt";
+
+ public IntTrieTokenStream(final int value, final int precisionStep) {
+ if (precisionStep<1 || precisionStep>32)
+ throw new IllegalArgumentException("precisionStep may only be 1..32");
+ this.value = value;
+ this.precisionStep = precisionStep;
+ termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+ typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
+ posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+ }
+
+ // @Override
+ public void reset() {
+ shift = 0;
+ termAtt.clear();
+ typeAtt.clear();
+ posIncrAtt.clear();
+ }
+
+ // @Override
+ public boolean incrementToken() {
+ if (shift>=32) return false;
+ final char[] buffer = termAtt.resizeTermBuffer(TrieUtils.INT_BUF_SIZE);
+ termAtt.setTermLength(TrieUtils.intToPrefixCoded(value, shift, buffer));
+ typeAtt.setType( (shift==0) ? TOKEN_TYPE_FULLPREC : TOKEN_TYPE_LOWERPREC );
+ posIncrAtt.setPositionIncrement( (shift==0) ? 1 : 0 );
+ shift += precisionStep;
+ return true;
+ }
+
+ // @Override
+ /** @deprecated */
+ public Token next(final Token reusableToken) {
+ assert reusableToken != null;
+ if (shift>=32) return null;
+ final char[] buffer = reusableToken.resizeTermBuffer(TrieUtils.INT_BUF_SIZE);
+ reusableToken.setTermLength(TrieUtils.intToPrefixCoded(value, shift, buffer));
+ reusableToken.setType( (shift==0) ? TOKEN_TYPE_FULLPREC : TOKEN_TYPE_LOWERPREC );
+ reusableToken.setPositionIncrement( (shift==0) ? 1 : 0 );
+ shift += precisionStep;
+ return reusableToken;
+ }
+
+ // members
+ private TermAttribute termAtt;
+ private TypeAttribute typeAtt;
+ private PositionIncrementAttribute posIncrAtt;
+
+ private int shift = 0;
+ private final int value;
+ private final int precisionStep;
+}
Index: contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java (revision 761586)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieRangeFilter.java (working copy)
@@ -49,46 +49,10 @@
public LongTrieRangeFilter(final String field, final int precisionStep,
final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
) {
- this(
- new String[]{field, field+TrieUtils.LOWER_PRECISION_FIELD_NAME_SUFFIX},
- precisionStep,min,max,minInclusive,maxInclusive
- );
+ super(field,precisionStep,min,max,minInclusive,maxInclusive);
}
-
- /**
- * Expert: A trie filter for matching trie coded values using the given field names.
- * You can specify the main and helper field name, that was used to idex the values.
- * precisionStep must me equal or a multiple of the precisionStep
- * used for indexing the values.
- * You can leave the bounds open, by supplying null for min and/or
- * max. Inclusive/exclusive bounds can also be supplied.
- * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
- *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String,String,String[])}.
- */
- public LongTrieRangeFilter(final String field, final String lowerPrecisionField, final int precisionStep,
- final Long min, final Long max, final boolean minInclusive, final boolean maxInclusive
- ) {
- this(new String[]{field, lowerPrecisionField},precisionStep,min,max,minInclusive,maxInclusive);
- }
/**
- * Expert: A trie filter for matching trie coded values
- * using the given field names. If the array of field names is shorter than the
- * trieCoded one, all trieCoded values with higher index get the last field name.
- * precisionStep must me equal or a multiple of the precisionStep
- * used for indexing the values.
- * You can leave the bounds open, by supplying null for min and/or
- * max. Inclusive/exclusive bounds can also be supplied.
- * To query double values use the converter {@link TrieUtils#doubleToSortableLong}.
- *
This is the counterpart to {@link TrieUtils#addIndexedFields(Document,String[],String[])}.
- */
- public LongTrieRangeFilter(final String[] fields, final int precisionStep,
- Long min, Long max, final boolean minInclusive, final boolean maxInclusive
- ) {
- super(fields, precisionStep, min, max, minInclusive, maxInclusive);
- }
-
- /**
* Returns a DocIdSet that provides the documents which should be permitted or prohibited in search results.
*/
//@Override
@@ -116,7 +80,6 @@
try {
fillBits(
reader, bits, termDocs,
- fields[Math.min(fields.length-1, level)],
minPrefixCoded, maxPrefixCoded
);
} catch (IOException ioe) {
Index: contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java (revision 0)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/LongTrieTokenStream.java (revision 0)
@@ -0,0 +1,81 @@
+package org.apache.lucene.search.trie;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
+public class LongTrieTokenStream extends TokenStream {
+
+ public static final String TOKEN_TYPE_FULLPREC = "fullPrecTrieLong";
+ public static final String TOKEN_TYPE_LOWERPREC = "lowerPrecTrieLong";
+
+ public LongTrieTokenStream(final long value, final int precisionStep) {
+ if (precisionStep<1 || precisionStep>64)
+ throw new IllegalArgumentException("precisionStep may only be 1..64");
+ this.value = value;
+ this.precisionStep = precisionStep;
+ termAtt = (TermAttribute) addAttribute(TermAttribute.class);
+ typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class);
+ posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
+ }
+
+ // @Override
+ public void reset() {
+ shift = 0;
+ termAtt.clear();
+ typeAtt.clear();
+ posIncrAtt.clear();
+ }
+
+ // @Override
+ public boolean incrementToken() {
+ if (shift>=64) return false;
+ final char[] buffer = termAtt.resizeTermBuffer(TrieUtils.LONG_BUF_SIZE);
+ termAtt.setTermLength(TrieUtils.longToPrefixCoded(value, shift, buffer));
+ typeAtt.setType( (shift==0) ? TOKEN_TYPE_FULLPREC : TOKEN_TYPE_LOWERPREC );
+ posIncrAtt.setPositionIncrement( (shift==0) ? 1 : 0 );
+ shift += precisionStep;
+ return true;
+ }
+
+ // @Override
+ /** @deprecated */
+ public Token next(final Token reusableToken) {
+ assert reusableToken != null;
+ if (shift>=64) return null;
+ final char[] buffer = reusableToken.resizeTermBuffer(TrieUtils.LONG_BUF_SIZE);
+ reusableToken.setTermLength(TrieUtils.longToPrefixCoded(value, shift, buffer));
+ reusableToken.setType( (shift==0) ? TOKEN_TYPE_FULLPREC : TOKEN_TYPE_LOWERPREC );
+ reusableToken.setPositionIncrement( (shift==0) ? 1 : 0 );
+ shift += precisionStep;
+ return reusableToken;
+ }
+
+ // members
+ private TermAttribute termAtt;
+ private TypeAttribute typeAtt;
+ private PositionIncrementAttribute posIncrAtt;
+
+ private int shift = 0;
+ private final long value;
+ private final int precisionStep;
+}
Index: contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java
===================================================================
--- contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (revision 761586)
+++ contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (working copy)
@@ -59,30 +59,31 @@
private TrieUtils() {} // no instance!
/**
- * The default "helper" field containing the lower precision terms is the original
- * fieldname with this appended. This suffix is used in
- * {@link #addIndexedFields(Document,String,String[])} and the corresponding c'tor
- * of (Long|Int)TrieRangeFilter.
- */
- public static final String LOWER_PRECISION_FIELD_NAME_SUFFIX="#trie";
-
- /**
* Longs are stored at lower precision by shifting off lower bits. The shift count is
* stored as SHIFT_START_LONG+shift in the first character
*/
public static final char SHIFT_START_LONG = (char)0x20;
+ /** internal */
+ static final int LONG_BUF_SIZE = 63/7 + 2;
+
/**
* Integers are stored at lower precision by shifting off lower bits. The shift count is
* stored as SHIFT_START_INT+shift in the first character
*/
public static final char SHIFT_START_INT = (char)0x60;
+ /** internal */
+ static final int INT_BUF_SIZE = 31/7 + 2;
+
/**
* A parser instance for filling a {@link ExtendedFieldCache}, that parses prefix encoded fields as longs.
*/
public static final ExtendedFieldCache.LongParser FIELD_CACHE_LONG_PARSER=new ExtendedFieldCache.LongParser(){
public final long parseLong(final String val) {
+ final int shift = val.charAt(0)-SHIFT_START_LONG;
+ if (shift>0 && shift<=63)
+ throw new FieldCache.StopFillCacheException();
return prefixCodedToLong(val);
}
};
@@ -92,6 +93,9 @@
*/
public static final FieldCache.IntParser FIELD_CACHE_INT_PARSER=new FieldCache.IntParser(){
public final int parseInt(final String val) {
+ final int shift = val.charAt(0)-SHIFT_START_INT;
+ if (shift>0 && shift<=31)
+ throw new FieldCache.StopFillCacheException();
return prefixCodedToInt(val);
}
};
@@ -102,6 +106,9 @@
*/
public static final ExtendedFieldCache.DoubleParser FIELD_CACHE_DOUBLE_PARSER=new ExtendedFieldCache.DoubleParser(){
public final double parseDouble(final String val) {
+ final int shift = val.charAt(0)-SHIFT_START_LONG;
+ if (shift>0 && shift<=63)
+ throw new FieldCache.StopFillCacheException();
return sortableLongToDouble(prefixCodedToLong(val));
}
};
@@ -112,6 +119,9 @@
*/
public static final FieldCache.FloatParser FIELD_CACHE_FLOAT_PARSER=new FieldCache.FloatParser(){
public final float parseFloat(final String val) {
+ final int shift = val.charAt(0)-SHIFT_START_INT;
+ if (shift>0 && shift<=31)
+ throw new FieldCache.StopFillCacheException();
return sortableIntToFloat(prefixCodedToInt(val));
}
};
@@ -125,7 +135,23 @@
public static String longToPrefixCoded(final long val) {
return longToPrefixCoded(val, 0);
}
-
+
+ /** internal */
+ static int longToPrefixCoded(final long val, final int shift, final char[] buffer) {
+ int nChars = (63-shift)/7 + 1, len = nChars+1;
+ buffer[0] = (char)(SHIFT_START_LONG + shift);
+ long sortableBits = val ^ 0x8000000000000000L;
+ sortableBits >>>= shift;
+ while (nChars>=1) {
+ // Store 7 bits per character for good efficiency when UTF-8 encoding.
+ // The whole number is right-justified so that lucene can prefix-encode
+ // the terms more efficiently.
+ buffer[nChars--] = (char)(sortableBits & 0x7f);
+ sortableBits >>>= 7;
+ }
+ return len;
+ }
+
/**
* Expert: Returns prefix coded bits after reducing the precision by shift bits.
* This is method is used by {@link #trieCodeLong}.
@@ -133,19 +159,25 @@
public static String longToPrefixCoded(final long val, final int shift) {
if (shift>63 || shift<0)
throw new IllegalArgumentException("Illegal shift value, must be 0..63");
- int nChars = (63-shift)/7 + 1;
- final char[] arr = new char[nChars+1];
- arr[0] = (char)(SHIFT_START_LONG + shift);
- long sortableBits = val ^ 0x8000000000000000L;
+ final char[] buffer = new char[LONG_BUF_SIZE];
+ final int len = longToPrefixCoded(val, shift, buffer);
+ return new String(buffer, 0, len);
+ }
+
+ /** internal */
+ static int intToPrefixCoded(final int val, final int shift, final char[] buffer) {
+ int nChars = (31-shift)/7 + 1, len = nChars+1;
+ buffer[0] = (char)(SHIFT_START_INT + shift);
+ int sortableBits = val ^ 0x80000000;
sortableBits >>>= shift;
while (nChars>=1) {
// Store 7 bits per character for good efficiency when UTF-8 encoding.
// The whole number is right-justified so that lucene can prefix-encode
// the terms more efficiently.
- arr[nChars--] = (char)(sortableBits & 0x7f);
+ buffer[nChars--] = (char)(sortableBits & 0x7f);
sortableBits >>>= 7;
}
- return new String(arr);
+ return len;
}
/**
@@ -165,19 +197,9 @@
public static String intToPrefixCoded(final int val, final int shift) {
if (shift>31 || shift<0)
throw new IllegalArgumentException("Illegal shift value, must be 0..31");
- int nChars = (31-shift)/7 + 1;
- final char[] arr = new char[nChars+1];
- arr[0] = (char)(SHIFT_START_INT + shift);
- int sortableBits = val ^ 0x80000000;
- sortableBits >>>= shift;
- while (nChars>=1) {
- // Store 7 bits per character for good efficiency when UTF-8 encoding.
- // The whole number is right-justified so that lucene can prefix-encode
- // the terms more efficiently.
- arr[nChars--] = (char)(sortableBits & 0x7f);
- sortableBits >>>= 7;
- }
- return new String(arr);
+ final char[] buffer = new char[INT_BUF_SIZE];
+ final int len = intToPrefixCoded(val, shift, buffer);
+ return new String(buffer, 0, len);
}
/**
@@ -293,7 +315,7 @@
* remaining values into another field.
*
To achieve this, use {@link #addIndexedFields(Document,String,String[])}. */ - public static String[] trieCodeLong(long val, int precisionStep) { + /*public static String[] trieCodeLong(long val, int precisionStep) { if (precisionStep<1 || precisionStep>64) throw new IllegalArgumentException("precisionStep may only be 1..64"); String[] arr = new String[63/precisionStep+1]; @@ -302,7 +324,7 @@ arr[idx++] = longToPrefixCoded(val, shift); } return arr; - } + }*/ /** * Returns a sequence of trie coded numbers suitable for {@link IntTrieRangeFilter}. @@ -311,7 +333,7 @@ * remaining values into another field. *
To achieve this, use {@link #addIndexedFields(Document,String,String[])}.
*/
- public static String[] trieCodeInt(int val, int precisionStep) {
+ /*public static String[] trieCodeInt(int val, int precisionStep) {
if (precisionStep<1 || precisionStep>32)
throw new IllegalArgumentException("precisionStep may only be 1..32");
String[] arr = new String[31/precisionStep+1];
@@ -320,67 +342,23 @@
arr[idx++] = intToPrefixCoded(val, shift);
}
return arr;
- }
+ }*/
/**
- * Indexes the full precision value only in the main field (for sorting), and indexes all other
- * lower precision values in field+LOWER_PRECISION_FIELD_NAME_SUFFIX.
- *
This is the recommended variant to add trie fields to the index.
- * By this it is possible to sort the field using a SortField instance
- * returned by {@link #getLongSortField} or {@link #getIntSortField}.
+ * Indexes a series of trie coded values into a lucene {@link Document}
+ * using the given field name.
*
This method does not store the fields and saves no term frequency or norms
* (which are normally not needed for trie fields). If you want to additionally store
* the value, you can use the normal methods of {@link Document} to achive this, just specify
- * Field.Store.YES, Field.Index.NO and the same field name.
- *
Examples: - *
- * addIndexedFields(doc, "mydouble", trieCodeLong(doubleToSortableLong(1.414d), 4)); - * addIndexedFields(doc, "mylong", trieCodeLong(123456L, 4)); - *- **/ - public static void addIndexedFields(Document doc, String field, String[] trieCoded) { - addIndexedFields(doc, new String[]{field, field+LOWER_PRECISION_FIELD_NAME_SUFFIX}, trieCoded); - } - - /** - * Expert: Indexes the full precision value only in the main field (for sorting), and indexes all other - * lower precision values in the
lowerPrecision field.
- * If you do not specify the same field name for the main and lower precision one,
- * it is possible to sort the field using a SortField instance
- * returned by {@link #getLongSortField} or {@link #getIntSortField}.
- * This method does not store the fields and saves no term frequency or norms
- * (which are normally not needed for trie fields). If you want to additionally store
- * the value, you can use the normal methods of {@link Document} to achive this, just specify
* Field.Store.YES, Field.Index.NO and the same main field name.
- *
Examples: - *
- * addIndexedFields(doc, "mydouble", "mydoubletrie", trieCodeLong(doubleToSortableLong(1.414d), 4)); - * addIndexedFields(doc, "mylong", "mylongtrie", trieCodeLong(123456L, 4)); - *- * @see #addIndexedFields(Document,String,String[]) **/ - public static void addIndexedFields(Document doc, String field, String lowerPrecisionField, String[] trieCoded) { - addIndexedFields(doc, new String[]{field, lowerPrecisionField}, trieCoded); - } - - /** - * Expert: Indexes a series of trie coded values into a lucene {@link Document} - * using the given field names. - * If the array of field names is shorter than the trie coded one, all trie coded - * values with higher index get the last field name. - *
This method does not store the fields and saves no term frequency or norms
- * (which are normally not needed for trie fields). If you want to additionally store
- * the value, you can use the normal methods of {@link Document} to achive this, just specify
- * DO NOT USE IN YOUR OWN CODE! It is just public
+ * to be accessible from o.a.l.search.trie.
+ */
+ public static class StopFillCacheException extends RuntimeException {
+ public StopFillCacheException() {
+ super();
+ }
+ }
+
/** Interface to parse bytes from document fields.
* @see FieldCache#getBytes(IndexReader, String, FieldCache.ByteParser)
*/
Index: src/java/org/apache/lucene/search/FieldCacheImpl.java
===================================================================
--- src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 761586)
+++ src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy)
@@ -196,6 +196,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
@@ -235,6 +236,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
@@ -274,6 +276,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
@@ -315,6 +318,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
Field.Store.YES, Field.Index.NO and the same main field name.
- **/
- public static void addIndexedFields(Document doc, String[] fields, String[] trieCoded) {
+ /*public static void addIndexedFields(Document doc, String field, String[] trieCoded) {
for (int i=0; i
- * String field = fields[Math.min(fields.length-1, level)];
- *
*/
public void addRange(String minPrefixCoded, String maxPrefixCoded, int level) {
throw new UnsupportedOperationException();
Index: contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java
===================================================================
--- contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java (revision 761586)
+++ contrib/queries/src/test/org/apache/lucene/search/trie/TestIntTrieRangeFilter.java (working copy)
@@ -42,6 +42,18 @@
// number of docs to generate for testing
private static final int noDocs = 10000;
+ private static void addField(Document doc, String name, int val, int precisionStep, boolean store) {
+ IntTrieTokenStream stream = new IntTrieTokenStream(val, precisionStep);
+ stream.setUseNewAPI(true);
+ Field f=new Field(name, stream);
+ f.setOmitTermFreqAndPositions(true);
+ f.setOmitNorms(true);
+ doc.add(f);
+ if (store) {
+ doc.add(new Field(name, TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+ }
+ }
+
private static final RAMDirectory directory;
private static final IndexSearcher searcher;
static {
@@ -55,16 +67,13 @@
Document doc=new Document();
// add fields, that have a distance to test general functionality
final int val=distance*l+startOffset;
- TrieUtils.addIndexedFields(doc,"field8", TrieUtils.trieCodeInt(val, 8));
- doc.add(new Field("field8", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
- TrieUtils.addIndexedFields(doc,"field4", TrieUtils.trieCodeInt(val, 4));
- doc.add(new Field("field4", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
- TrieUtils.addIndexedFields(doc,"field2", TrieUtils.trieCodeInt(val, 2));
- doc.add(new Field("field2", TrieUtils.intToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+ addField(doc,"field8", val, 8, true);
+ addField(doc,"field4", val, 4, true);
+ addField(doc,"field2", val, 2, true);
// add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
- TrieUtils.addIndexedFields(doc,"ascfield8", TrieUtils.trieCodeInt(l-(noDocs/2), 8));
- TrieUtils.addIndexedFields(doc,"ascfield4", TrieUtils.trieCodeInt(l-(noDocs/2), 4));
- TrieUtils.addIndexedFields(doc,"ascfield2", TrieUtils.trieCodeInt(l-(noDocs/2), 2));
+ addField(doc,"ascfield8", l-(noDocs/2), 8, false);
+ addField(doc,"ascfield4", l-(noDocs/2), 4, false);
+ addField(doc,"ascfield2", l-(noDocs/2), 2, false);
writer.addDocument(doc);
}
Index: contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java
===================================================================
--- contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java (revision 761586)
+++ contrib/queries/src/test/org/apache/lucene/search/trie/TestLongTrieRangeFilter.java (working copy)
@@ -42,6 +42,18 @@
// number of docs to generate for testing
private static final int noDocs = 10000;
+ private static void addField(Document doc, String name, long val, int precisionStep, boolean store) {
+ LongTrieTokenStream stream = new LongTrieTokenStream(val, precisionStep);
+ stream.setUseNewAPI(true);
+ Field f=new Field(name, stream);
+ f.setOmitTermFreqAndPositions(true);
+ f.setOmitNorms(true);
+ doc.add(f);
+ if (store) {
+ doc.add(new Field(name, TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+ }
+ }
+
private static final RAMDirectory directory;
private static final IndexSearcher searcher;
static {
@@ -55,16 +67,13 @@
Document doc=new Document();
// add fields, that have a distance to test general functionality
final long val=distance*l+startOffset;
- TrieUtils.addIndexedFields(doc,"field8", TrieUtils.trieCodeLong(val, 8));
- doc.add(new Field("field8", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
- TrieUtils.addIndexedFields(doc,"field4", TrieUtils.trieCodeLong(val, 4));
- doc.add(new Field("field4", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
- TrieUtils.addIndexedFields(doc,"field2", TrieUtils.trieCodeLong(val, 2));
- doc.add(new Field("field2", TrieUtils.longToPrefixCoded(val), Field.Store.YES, Field.Index.NO));
+ addField(doc,"field8", val, 8, true);
+ addField(doc,"field4", val, 4, true);
+ addField(doc,"field2", val, 2, true);
// add ascending fields with a distance of 1, beginning at -noDocs/2 to test the correct splitting of range and inclusive/exclusive
- TrieUtils.addIndexedFields(doc,"ascfield8", TrieUtils.trieCodeLong(l-(noDocs/2), 8));
- TrieUtils.addIndexedFields(doc,"ascfield4", TrieUtils.trieCodeLong(l-(noDocs/2), 4));
- TrieUtils.addIndexedFields(doc,"ascfield2", TrieUtils.trieCodeLong(l-(noDocs/2), 2));
+ addField(doc,"ascfield8", l-(noDocs/2), 8, false);
+ addField(doc,"ascfield4", l-(noDocs/2), 4, false);
+ addField(doc,"ascfield2", l-(noDocs/2), 2, false);
writer.addDocument(doc);
}
Index: src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java
===================================================================
--- src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java (revision 761586)
+++ src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java (working copy)
@@ -73,6 +73,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
@@ -113,6 +114,7 @@
retArray[termDocs.doc()] = termval;
}
} while (termEnum.next());
+ } catch (StopFillCacheException stop) {
} finally {
termDocs.close();
termEnum.close();
Index: src/java/org/apache/lucene/search/FieldCache.java
===================================================================
--- src/java/org/apache/lucene/search/FieldCache.java (revision 761586)
+++ src/java/org/apache/lucene/search/FieldCache.java (working copy)
@@ -82,6 +82,18 @@
public interface Parser {
}
+ /**
+ * Internal hack for contrib's TrieUtils, to stop parsing,
+ * when lower precision terms appear during parsing.
+ *