Index: lucene/src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FieldCache.java (revision 1172228) +++ lucene/src/java/org/apache/lucene/search/FieldCache.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.document.NumericField; // for javadocs import org.apache.lucene.analysis.NumericTokenStream; // for javadocs +import org.apache.lucene.util.FixedBitSet; import java.io.IOException; import java.io.Serializable; @@ -315,7 +316,7 @@ * reader.maxDoc(), with turned on bits for each docid that * does not have a value for this field. */ - public DocIdSet getUnValuedDocs (IndexReader reader, String field) + public FixedBitSet getUnValuedDocs (IndexReader reader, String field) throws IOException; /** Checks the internal cache for an appropriate entry, and if none is Index: lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 1172228) +++ lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -32,7 +32,7 @@ import org.apache.lucene.index.TermEnum; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.DocIdBitSet; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.StringHelper; import org.apache.lucene.util.FieldCacheSanityChecker; @@ -419,17 +419,20 @@ super(wrapper); } + final static Object EMPTY_PLACEHOLDER = new Object(); + @Override protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = entryKey; String field = entry.field; + // TODO (please verify): Can this ever work???? ....unlikely: if (reader.maxDoc() == reader.docFreq(new Term(field))) { - return DocIdSet.EMPTY_DOCIDSET; + return EMPTY_PLACEHOLDER; } - OpenBitSet res = new OpenBitSet(reader.maxDoc()); + FixedBitSet res = new FixedBitSet(reader.maxDoc()); TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms (new Term (field)); try { @@ -438,7 +441,7 @@ if (term==null || term.field() != field) break; termDocs.seek (termEnum); while (termDocs.next()) { - res.fastSet(termDocs.doc()); + res.set(termDocs.doc()); } } while (termEnum.next()); } finally { @@ -446,7 +449,7 @@ termEnum.close(); } res.flip(0, reader.maxDoc()); - return res; + return (res.cardinality() == 0) ? EMPTY_PLACEHOLDER : res; } } @@ -737,9 +740,10 @@ return infoStream; } - public DocIdSet getUnValuedDocs(IndexReader reader, String field) + public FixedBitSet getUnValuedDocs(IndexReader reader, String field) throws IOException { - return (DocIdSet) caches.get(UnValuedDocsCache.class).get(reader, new Entry(field, null)); + final Object o = caches.get(UnValuedDocsCache.class).get(reader, new Entry(field, null)); + return (o == UnValuedDocsCache.EMPTY_PLACEHOLDER) ? null : (FixedBitSet) o; } } Index: lucene/src/java/org/apache/lucene/search/FieldComparator.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FieldComparator.java (revision 1172228) +++ lucene/src/java/org/apache/lucene/search/FieldComparator.java (working copy) @@ -29,7 +29,7 @@ import org.apache.lucene.search.FieldCache.IntParser; import org.apache.lucene.search.FieldCache.ShortParser; import org.apache.lucene.search.FieldCache.StringIndex; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FixedBitSet; /** * Expert: a FieldComparator compares hits so as to determine their @@ -84,14 +84,6 @@ */ public abstract class FieldComparator { - protected T missingValue = null; - - /** Set a default sorting value for documents which lacks one */ - public FieldComparator setMissingValue(T missingValue) { - this.missingValue = missingValue; - return this; - } - /** * Compare hit at slot1 with hit at slot2. * @@ -191,18 +183,41 @@ } } + public static abstract class NumericComparator extends FieldComparator { + protected final T missingValue; + protected final String field; + + protected FixedBitSet unvaluedDocs = null; + + public NumericComparator(String field, T missingValue) { + this.field = field; + this.missingValue = missingValue; + } + + @Override + public void setNextReader(IndexReader reader, int docBase) throws IOException { + unvaluedDocs = (missingValue != null) ? + FieldCache.DEFAULT.getUnValuedDocs(reader, field) : + null; + } + + } + /** Parses field's values as byte (using {@link * FieldCache#getBytes} and sorts by ascending value */ - public static final class ByteComparator extends FieldComparator { + public static final class ByteComparator extends NumericComparator { private final byte[] values; private byte[] currentReaderValues; - private final String field; private ByteParser parser; private byte bottom; ByteComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + ByteComparator(int numHits, String field, FieldCache.Parser parser, Byte missingValue) { + super(field, missingValue); values = new byte[numHits]; - this.field = field; this.parser = (ByteParser) parser; } @@ -213,24 +228,24 @@ @Override public int compareBottom(int doc) { - return bottom - currentReaderValues[doc]; + byte v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + return bottom - v2; } @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + byte v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getBytes(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final byte byteValue = missingValue.byteValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = byteValue; - } - } + super.setNextReader(reader, docBase); } @Override @@ -292,16 +307,19 @@ /** Parses field's values as double (using {@link * FieldCache#getDoubles} and sorts by ascending value */ - public static final class DoubleComparator extends FieldComparator { + public static final class DoubleComparator extends NumericComparator { private final double[] values; private double[] currentReaderValues; - private final String field; private DoubleParser parser; private double bottom; DoubleComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + DoubleComparator(int numHits, String field, FieldCache.Parser parser, Double missingValue) { + super(field, missingValue); values = new double[numHits]; - this.field = field; this.parser = (DoubleParser) parser; } @@ -320,7 +338,9 @@ @Override public int compareBottom(int doc) { - final double v2 = currentReaderValues[doc]; + double v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; if (bottom > v2) { return 1; } else if (bottom < v2) { @@ -332,19 +352,16 @@ @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + double v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getDoubles(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final double doubleValue = missingValue.doubleValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = doubleValue; - } - } + super.setNextReader(reader, docBase); } @Override @@ -360,16 +377,19 @@ /** Parses field's values as float (using {@link * FieldCache#getFloats} and sorts by ascending value */ - public static final class FloatComparator extends FieldComparator { + public static final class FloatComparator extends NumericComparator { private final float[] values; private float[] currentReaderValues; - private final String field; private FloatParser parser; private float bottom; FloatComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + FloatComparator(int numHits, String field, FieldCache.Parser parser, Float missingValue) { + super(field, missingValue); values = new float[numHits]; - this.field = field; this.parser = (FloatParser) parser; } @@ -392,7 +412,9 @@ public int compareBottom(int doc) { // TODO: are there sneaky non-branch ways to compute // sign of float? - final float v2 = currentReaderValues[doc]; + float v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; if (bottom > v2) { return 1; } else if (bottom < v2) { @@ -404,19 +426,16 @@ @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + float v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getFloats(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final float floatValue = missingValue.floatValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = floatValue; - } - } + super.setNextReader(reader, docBase); } @Override @@ -432,16 +451,19 @@ /** Parses field's values as int (using {@link * FieldCache#getInts} and sorts by ascending value */ - public static final class IntComparator extends FieldComparator { + public static final class IntComparator extends NumericComparator { private final int[] values; private int[] currentReaderValues; - private final String field; private IntParser parser; private int bottom; // Value of bottom of queue IntComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + IntComparator(int numHits, String field, FieldCache.Parser parser, Integer missingValue) { + super(field, missingValue); values = new int[numHits]; - this.field = field; this.parser = (IntParser) parser; } @@ -468,7 +490,9 @@ // -1/+1/0 sign // Cannot return bottom - values[slot2] because that // may overflow - final int v2 = currentReaderValues[doc]; + int v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; if (bottom > v2) { return 1; } else if (bottom < v2) { @@ -480,19 +504,16 @@ @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + int v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final int intValue = missingValue.intValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = intValue; - } - } + super.setNextReader(reader, docBase); } @Override @@ -508,16 +529,19 @@ /** Parses field's values as long (using {@link * FieldCache#getLongs} and sorts by ascending value */ - public static final class LongComparator extends FieldComparator { + public static final class LongComparator extends NumericComparator { private final long[] values; private long[] currentReaderValues; - private final String field; private LongParser parser; private long bottom; LongComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + LongComparator(int numHits, String field, FieldCache.Parser parser, Long missingValue) { + super(field, missingValue); values = new long[numHits]; - this.field = field; this.parser = (LongParser) parser; } @@ -540,7 +564,9 @@ public int compareBottom(int doc) { // TODO: there are sneaky non-branch ways to compute // -1/+1/0 sign - final long v2 = currentReaderValues[doc]; + long v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; if (bottom > v2) { return 1; } else if (bottom < v2) { @@ -552,19 +578,16 @@ @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + long v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getLongs(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final long longValue = missingValue.longValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = longValue; - } - } + super.setNextReader(reader, docBase); } @Override @@ -648,16 +671,19 @@ /** Parses field's values as short (using {@link * FieldCache#getShorts} and sorts by ascending value */ - public static final class ShortComparator extends FieldComparator { + public static final class ShortComparator extends NumericComparator { private final short[] values; private short[] currentReaderValues; - private final String field; private ShortParser parser; private short bottom; ShortComparator(int numHits, String field, FieldCache.Parser parser) { + this(numHits, field, parser, null); + } + + ShortComparator(int numHits, String field, FieldCache.Parser parser, Short missingValue) { + super(field, missingValue); values = new short[numHits]; - this.field = field; this.parser = (ShortParser) parser; } @@ -668,24 +694,24 @@ @Override public int compareBottom(int doc) { - return bottom - currentReaderValues[doc]; + short v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + return bottom - v2; } @Override public void copy(int slot, int doc) { - values[slot] = currentReaderValues[doc]; + short v2 = currentReaderValues[doc]; + if (unvaluedDocs != null && v2 == 0 && unvaluedDocs.get(doc)) + v2 = missingValue; + values[slot] = v2; } @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getShorts(reader, field, parser); - if (missingValue != null) { - DocIdSetIterator iterator = FieldCache.DEFAULT.getUnValuedDocs(reader, field).iterator(); - final short shortValue = missingValue.shortValue(); - for (int doc=iterator.nextDoc(); doc!=DocIdSetIterator.NO_MORE_DOCS; doc=iterator.nextDoc()) { - currentReaderValues[doc] = shortValue; - } - } + super.setNextReader(reader, docBase); } @Override Index: lucene/src/java/org/apache/lucene/search/SortField.java =================================================================== --- lucene/src/java/org/apache/lucene/search/SortField.java (revision 1172228) +++ lucene/src/java/org/apache/lucene/search/SortField.java (working copy) @@ -403,22 +403,22 @@ return new FieldComparator.DocComparator(numHits); case SortField.INT: - return new FieldComparator.IntComparator(numHits, field, parser).setMissingValue((Integer) missingValue); + return new FieldComparator.IntComparator(numHits, field, parser, (Integer) missingValue); case SortField.FLOAT: - return new FieldComparator.FloatComparator(numHits, field, parser).setMissingValue((Float) missingValue); + return new FieldComparator.FloatComparator(numHits, field, parser, (Float) missingValue); case SortField.LONG: - return new FieldComparator.LongComparator(numHits, field, parser).setMissingValue((Long) missingValue); + return new FieldComparator.LongComparator(numHits, field, parser, (Long) missingValue); case SortField.DOUBLE: - return new FieldComparator.DoubleComparator(numHits, field, parser).setMissingValue((Double) missingValue); + return new FieldComparator.DoubleComparator(numHits, field, parser, (Double) missingValue); case SortField.BYTE: - return new FieldComparator.ByteComparator(numHits, field, parser).setMissingValue((Byte) missingValue); + return new FieldComparator.ByteComparator(numHits, field, parser, (Byte) missingValue); case SortField.SHORT: - return new FieldComparator.ShortComparator(numHits, field, parser).setMissingValue((Short) missingValue); + return new FieldComparator.ShortComparator(numHits, field, parser, (Short) missingValue); case SortField.CUSTOM: assert comparatorSource != null;