Index: lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java (revision 1337911) +++ lucene/core/src/test/org/apache/lucene/search/TestSearchAfter.java (working copy) @@ -17,12 +17,25 @@ * limitations under the License. */ +import java.util.Arrays; + +import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.FloatDocValuesField; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntDocValuesField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.SortedBytesDocValuesField; +import org.apache.lucene.document.StraightBytesDocValuesField; +import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.English; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -30,11 +43,19 @@ /** * Tests IndexSearcher's searchAfter() method */ + public class TestSearchAfter extends LuceneTestCase { private Directory dir; private IndexReader reader; private IndexSearcher searcher; + boolean supportsDocValues = Codec.getDefault().getName().equals("Lucene3x") == false; + + private static SortField useDocValues(SortField field) { + field.setUseIndexValues(true); + return field; + } + @Override public void setUp() throws Exception { super.setUp(); @@ -45,6 +66,25 @@ Document document = new Document(); document.add(newField("english", English.intToEnglish(i), TextField.TYPE_UNSTORED)); document.add(newField("oddeven", (i % 2 == 0) ? "even" : "odd", TextField.TYPE_UNSTORED)); + document.add(newField("byte", "" + ((byte) random().nextInt()), StringField.TYPE_UNSTORED)); + document.add(newField("short", "" + ((short) random().nextInt()), StringField.TYPE_UNSTORED)); + document.add(new IntField("int", random().nextInt())); + document.add(new LongField("long", random().nextLong())); + + document.add(new FloatField("float", random().nextFloat())); + document.add(new DoubleField("double", random().nextDouble())); + document.add(newField("bytes", _TestUtil.randomRealisticUnicodeString(random()), StringField.TYPE_UNSTORED)); + document.add(newField("bytesval", _TestUtil.randomRealisticUnicodeString(random()), StringField.TYPE_UNSTORED)); + document.add(new DoubleField("double", random().nextDouble())); + + if (supportsDocValues) { + document.add(new IntDocValuesField("intdocvalues", random().nextInt())); + document.add(new FloatDocValuesField("floatdocvalues", random().nextFloat())); + document.add(new SortedBytesDocValuesField("sortedbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); + document.add(new SortedBytesDocValuesField("sortedbytesdocvaluesval", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); + document.add(new StraightBytesDocValuesField("straightbytesdocvalues", new BytesRef(_TestUtil.randomRealisticUnicodeString(random())))); + } + iw.addDocument(document); } reader = iw.getReader(); @@ -63,7 +103,7 @@ // because the first page has a null 'after', we get a normal collector. // so we need to run the test a few times to ensure we will collect multiple // pages. - int n = atLeast(10); + int n = atLeast(20); for (int i = 0; i < n; i++) { Filter odd = new QueryWrapperFilter(new TermQuery(new Term("oddeven", "odd"))); assertQuery(new MatchAllDocsQuery(), null); @@ -78,13 +118,67 @@ } void assertQuery(Query query, Filter filter) throws Exception { + assertQuery(query, filter, null); + assertQuery(query, filter, Sort.RELEVANCE); + assertQuery(query, filter, Sort.INDEXORDER); + for(int rev=0;rev<2;rev++) { + boolean reversed = rev == 1; + assertQuery(query, filter, new Sort(new SortField[] {new SortField("byte", SortField.Type.BYTE, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("short", SortField.Type.SHORT, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("int", SortField.Type.INT, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("long", SortField.Type.LONG, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("float", SortField.Type.FLOAT, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("double", SortField.Type.DOUBLE, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytes", SortField.Type.STRING, reversed)})); + assertQuery(query, filter, new Sort(new SortField[] {new SortField("bytesval", SortField.Type.STRING_VAL, reversed)})); + if (supportsDocValues) { + assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("intdocvalues", SortField.Type.INT, reversed))})); + assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("floatdocvalues", SortField.Type.FLOAT, reversed))})); + assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("sortedbytesdocvalues", SortField.Type.STRING, reversed))})); + assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("sortedbytesdocvaluesval", SortField.Type.STRING_VAL, reversed))})); + assertQuery(query, filter, new Sort(new SortField[] {useDocValues(new SortField("straightbytesdocvalues", SortField.Type.STRING_VAL, reversed))})); + } + } + } + + void assertQuery(Query query, Filter filter, Sort sort) throws Exception { int maxDoc = searcher.getIndexReader().maxDoc(); - TopDocs all = searcher.search(query, filter, maxDoc); + TopDocs all; int pageSize = _TestUtil.nextInt(random(), 1, maxDoc*2); + if (VERBOSE) { + System.out.println("\nassertQuery: query=" + query + " filter=" + filter + " sort=" + sort + " pageSize=" + pageSize); + } + final boolean doMaxScore = random().nextBoolean(); + if (sort == null) { + all = searcher.search(query, filter, maxDoc); + } else if (sort == Sort.RELEVANCE) { + all = searcher.search(query, filter, maxDoc, sort, true, doMaxScore); + } else { + all = searcher.search(query, filter, maxDoc, sort); + } + if (VERBOSE) { + System.out.println(" all.totalHits=" + all.totalHits); + } int pageStart = 0; ScoreDoc lastBottom = null; while (pageStart < all.totalHits) { - TopDocs paged = searcher.searchAfter(lastBottom, query, filter, pageSize); + TopDocs paged; + if (sort == null) { + if (VERBOSE) { + System.out.println(" iter lastBottom=" + lastBottom); + } + paged = searcher.searchAfter(lastBottom, query, filter, pageSize); + } else { + if (VERBOSE) { + System.out.println(" iter lastBottom=" + lastBottom + (lastBottom == null ? "" : " fields=" + Arrays.toString(((FieldDoc) lastBottom).fields))); + } + if (sort == Sort.RELEVANCE) { + paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort, true, doMaxScore); + } else { + paged = searcher.searchAfter(lastBottom, query, filter, pageSize, sort); + } + } + if (paged.scoreDocs.length == 0) { break; } @@ -98,8 +192,14 @@ static void assertPage(int pageStart, TopDocs all, TopDocs paged) { assertEquals(all.totalHits, paged.totalHits); for (int i = 0; i < paged.scoreDocs.length; i++) { - assertEquals(all.scoreDocs[pageStart + i].doc, paged.scoreDocs[i].doc); - assertEquals(all.scoreDocs[pageStart + i].score, paged.scoreDocs[i].score, 0f); + ScoreDoc sd1 = all.scoreDocs[pageStart + i]; + ScoreDoc sd2 = paged.scoreDocs[i]; + assertEquals(sd1.doc, sd2.doc); + assertEquals(sd1.score, sd2.score, 0f); + if (sd1 instanceof FieldDoc) { + assertTrue(sd2 instanceof FieldDoc); + assertEquals(((FieldDoc) sd1).fields, ((FieldDoc) sd2).fields); + } } } } Index: lucene/core/src/test/org/apache/lucene/search/TestSort.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSort.java (revision 1337911) +++ lucene/core/src/test/org/apache/lucene/search/TestSort.java (working copy) @@ -218,7 +218,6 @@ IndexReader reader = writer.getReader(); writer.close (); IndexSearcher s = newSearcher(reader); - s.setDefaultFieldSortScoring(true, true); return s; } @@ -734,6 +733,15 @@ public Integer value(int slot) { return Integer.valueOf(slotValues[slot]); } + + @Override + public int compareDocToValue(int doc, Integer valueObj) { + final int value = valueObj.intValue(); + final int docValue = docValues[doc]; + + // values are small enough that overflow won't happen + return docValue - value; + } } static class MyFieldComparatorSource extends FieldComparatorSource { Index: lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1337911) +++ lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -139,6 +139,10 @@ throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + @Override + public int compareDocToValue(int doc, Object value) { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } } static final class JustCompileFieldComparatorSource extends FieldComparatorSource { Index: lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java (revision 1337911) +++ lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java (working copy) @@ -188,6 +188,14 @@ public Integer value(int slot) { return Integer.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Integer valueObj) throws IOException { + final int value = valueObj.intValue(); + final int docValue = docVal(doc); + // values will be small enough that there is no overflow concern + return value - docValue; + } }; } } Index: lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java (revision 1337911) +++ lucene/core/src/java/org/apache/lucene/search/TopFieldCollector.java (working copy) @@ -843,6 +843,166 @@ } + /* + * Implements a TopFieldCollector when after != null. + */ + private final static class PagingFieldCollector extends TopFieldCollector { + + Scorer scorer; + int collectedHits; + final FieldComparator[] comparators; + final int[] reverseMul; + final FieldValueHitQueue queue; + final boolean trackDocScores; + final boolean trackMaxScore; + final FieldDoc after; + int afterDoc; + + public PagingFieldCollector( + FieldValueHitQueue queue, FieldDoc after, int numHits, boolean fillFields, + boolean trackDocScores, boolean trackMaxScore) + throws IOException { + super(queue, numHits, fillFields); + this.queue = queue; + this.trackDocScores = trackDocScores; + this.trackMaxScore = trackMaxScore; + this.after = after; + comparators = queue.getComparators(); + reverseMul = queue.getReverseMul(); + + // Must set maxScore to NEG_INF, or otherwise Math.max always returns NaN. + maxScore = Float.NEGATIVE_INFINITY; + } + + final void updateBottom(int doc, float score) { + bottom.doc = docBase + doc; + bottom.score = score; + bottom = pq.updateTop(); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Override + public void collect(int doc) throws IOException { + totalHits++; + + //System.out.println(" collect doc=" + doc); + + // Check if this hit was already collected on a + // previous page: + boolean sameValues = true; + for(int compIDX=0;compIDX 0) { + // Not yet collected + sameValues = false; + //System.out.println(" keep: after"); + break; + } + } + + // Tie-break by docID: + if (sameValues && doc <= afterDoc) { + // Already collected on a previous page + //System.out.println(" skip: tie-break"); + return; + } + + collectedHits++; + + float score = Float.NaN; + if (trackMaxScore) { + score = scorer.score(); + if (score > maxScore) { + maxScore = score; + } + } + + if (queueFull) { + // Fastmatch: return if this hit is not competitive + for (int i = 0;; i++) { + final int c = reverseMul[i] * comparators[i].compareBottom(doc); + if (c < 0) { + // Definitely not competitive. + return; + } else if (c > 0) { + // Definitely competitive. + break; + } else if (i == comparators.length - 1) { + // This is the equals case. + if (doc + docBase > bottom.doc) { + // Definitely not competitive + return; + } + break; + } + } + + // This hit is competitive - replace bottom element in queue & adjustTop + for (int i = 0; i < comparators.length; i++) { + comparators[i].copy(bottom.slot, doc); + } + + // Compute score only if it is competitive. + if (trackDocScores && !trackMaxScore) { + score = scorer.score(); + } + updateBottom(doc, score); + + for (int i = 0; i < comparators.length; i++) { + comparators[i].setBottom(bottom.slot); + } + } else { + // Startup transient: queue hasn't gathered numHits yet + final int slot = collectedHits - 1; + //System.out.println(" slot=" + slot); + // Copy hit into queue + for (int i = 0; i < comparators.length; i++) { + comparators[i].copy(slot, doc); + } + + // Compute score only if it is competitive. + if (trackDocScores && !trackMaxScore) { + score = scorer.score(); + } + bottom = pq.add(new Entry(slot, docBase + doc, score)); + queueFull = collectedHits == numHits; + if (queueFull) { + for (int i = 0; i < comparators.length; i++) { + comparators[i].setBottom(bottom.slot); + } + } + } + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + for (int i = 0; i < comparators.length; i++) { + comparators[i].setScorer(scorer); + } + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + docBase = context.docBase; + afterDoc = after.doc - docBase; + for (int i = 0; i < comparators.length; i++) { + queue.setComparator(i, comparators[i].setNextReader(context)); + } + } + } + private static final ScoreDoc[] EMPTY_SCOREDOCS = new ScoreDoc[0]; private final boolean fillFields; @@ -909,6 +1069,52 @@ boolean fillFields, boolean trackDocScores, boolean trackMaxScore, boolean docsScoredInOrder) throws IOException { + return create(sort, numHits, null, fillFields, trackDocScores, trackMaxScore, docsScoredInOrder); + } + + /** + * Creates a new {@link TopFieldCollector} from the given + * arguments. + * + *

NOTE: The instances returned by this method + * pre-allocate a full array of length + * numHits. + * + * @param sort + * the sort criteria (SortFields). + * @param numHits + * the number of results to collect. + * @param after + * only hits after this FieldDoc will be collected + * @param fillFields + * specifies whether the actual field values should be returned on + * the results (FieldDoc). + * @param trackDocScores + * specifies whether document scores should be tracked and set on the + * results. Note that if set to false, then the results' scores will + * be set to Float.NaN. Setting this to true affects performance, as + * it incurs the score computation on each competitive result. + * Therefore if document scores are not required by the application, + * it is recommended to set it to false. + * @param trackMaxScore + * specifies whether the query's maxScore should be tracked and set + * on the resulting {@link TopDocs}. Note that if set to false, + * {@link TopDocs#getMaxScore()} returns Float.NaN. Setting this to + * true affects performance as it incurs the score computation on + * each result. Also, setting this true automatically sets + * trackDocScores to true as well. + * @param docsScoredInOrder + * specifies whether documents are scored in doc Id order or not by + * the given {@link Scorer} in {@link #setScorer(Scorer)}. + * @return a {@link TopFieldCollector} instance which will sort the results by + * the sort criteria. + * @throws IOException + */ + public static TopFieldCollector create(Sort sort, int numHits, FieldDoc after, + boolean fillFields, boolean trackDocScores, boolean trackMaxScore, + boolean docsScoredInOrder) + throws IOException { + if (sort.fields.length == 0) { throw new IllegalArgumentException("Sort must contain at least one field"); } @@ -918,43 +1124,56 @@ } FieldValueHitQueue queue = FieldValueHitQueue.create(sort.fields, numHits); - if (queue.getComparators().length == 1) { + + if (after == null) { + if (queue.getComparators().length == 1) { + if (docsScoredInOrder) { + if (trackMaxScore) { + return new OneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + } else if (trackDocScores) { + return new OneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + } else { + return new OneComparatorNonScoringCollector(queue, numHits, fillFields); + } + } else { + if (trackMaxScore) { + return new OutOfOrderOneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + } else if (trackDocScores) { + return new OutOfOrderOneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + } else { + return new OutOfOrderOneComparatorNonScoringCollector(queue, numHits, fillFields); + } + } + } + + // multiple comparators. if (docsScoredInOrder) { if (trackMaxScore) { - return new OneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + return new MultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); } else if (trackDocScores) { - return new OneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + return new MultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); } else { - return new OneComparatorNonScoringCollector(queue, numHits, fillFields); + return new MultiComparatorNonScoringCollector(queue, numHits, fillFields); } } else { if (trackMaxScore) { - return new OutOfOrderOneComparatorScoringMaxScoreCollector(queue, numHits, fillFields); + return new OutOfOrderMultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); } else if (trackDocScores) { - return new OutOfOrderOneComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); + return new OutOfOrderMultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); } else { - return new OutOfOrderOneComparatorNonScoringCollector(queue, numHits, fillFields); + return new OutOfOrderMultiComparatorNonScoringCollector(queue, numHits, fillFields); } } - } + } else { + if (after.fields == null) { + throw new IllegalArgumentException("after.fields wasn't set; you must pass fillFields=true for the previous search"); + } - // multiple comparators. - if (docsScoredInOrder) { - if (trackMaxScore) { - return new MultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); - } else if (trackDocScores) { - return new MultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); - } else { - return new MultiComparatorNonScoringCollector(queue, numHits, fillFields); + if (after.fields.length != sort.getSort().length) { + throw new IllegalArgumentException("after.fields has " + after.fields.length + " values but sort has " + sort.getSort().length); } - } else { - if (trackMaxScore) { - return new OutOfOrderMultiComparatorScoringMaxScoreCollector(queue, numHits, fillFields); - } else if (trackDocScores) { - return new OutOfOrderMultiComparatorScoringNoMaxScoreCollector(queue, numHits, fillFields); - } else { - return new OutOfOrderMultiComparatorNonScoringCollector(queue, numHits, fillFields); - } + + return new PagingFieldCollector(queue, after, numHits, fillFields, trackDocScores, trackMaxScore); } } Index: lucene/core/src/java/org/apache/lucene/search/TopDocs.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TopDocs.java (revision 1337911) +++ lucene/core/src/java/org/apache/lucene/search/TopDocs.java (working copy) @@ -45,7 +45,7 @@ /** Sets the maximum score value encountered. */ public void setMaxScore(float maxScore) { - this.maxScore=maxScore; + this.maxScore = maxScore; } /** Constructs a TopDocs with a default maxScore=Float.NaN. */ Index: lucene/core/src/java/org/apache/lucene/search/FieldComparator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (revision 1337911) +++ lucene/core/src/java/org/apache/lucene/search/FieldComparator.java (working copy) @@ -190,6 +190,10 @@ } } + /** Returns negative result if the doc's value is less + * than the provided value. */ + public abstract int compareDocToValue(int doc, T value) throws IOException; + public static abstract class NumericComparator extends FieldComparator { protected final T missingValue; protected final String field; @@ -274,9 +278,19 @@ public Byte value(int slot) { return Byte.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Byte value) { + byte docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + return docValue - value.byteValue(); + } } - /** Parses field's values as double (using {@link * FieldCache#getDoubles} and sorts by ascending value */ public static final class DoubleComparator extends NumericComparator { @@ -351,6 +365,24 @@ public Double value(int slot) { return Double.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Double valueObj) { + final double value = valueObj.doubleValue(); + double docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Uses float index values to sort by ascending value */ @@ -415,6 +447,19 @@ public Double value(int slot) { return Double.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Double valueObj) { + final double value = valueObj.doubleValue(); + final double docValue = currentReaderValues.getFloat(doc); + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Parses field's values as float (using {@link @@ -494,6 +539,24 @@ public Float value(int slot) { return Float.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Float valueObj) { + final float value = valueObj.floatValue(); + float docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Parses field's values as short (using {@link @@ -556,6 +619,18 @@ public Short value(int slot) { return Short.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Short valueObj) { + final short value = valueObj.shortValue(); + short docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + return docValue - value; + } } /** Parses field's values as int (using {@link @@ -640,6 +715,24 @@ public Integer value(int slot) { return Integer.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Integer valueObj) { + final int value = valueObj.intValue(); + int docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Loads int index values and sorts by ascending value. */ @@ -708,6 +801,19 @@ public Long value(int slot) { return Long.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Long valueObj) { + final long value = valueObj.longValue(); + final long docValue = currentReaderValues.getInt(doc); + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Parses field's values as long (using {@link @@ -788,6 +894,24 @@ public Long value(int slot) { return Long.valueOf(values[slot]); } + + @Override + public int compareDocToValue(int doc, Long valueObj) { + final long value = valueObj.longValue(); + long docValue = currentReaderValues[doc]; + // Test for docValue == 0 to save Bits.get method call for + // the common case (doc has value and value is non-zero): + if (docsWithField != null && docValue == 0 && !docsWithField.get(doc)) { + docValue = missingValue; + } + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Sorts by descending relevance. NOTE: if you are @@ -815,12 +939,19 @@ @Override public int compareBottom(int doc) throws IOException { float score = scorer.score(); + // TODO: should we assert this...? can't compare + // NaNs... + // nocommit + assert !Float.isNaN(score); return bottom > score ? -1 : (bottom < score ? 1 : 0); } @Override public void copy(int slot, int doc) throws IOException { scores[slot] = scorer.score(); + // TODO: should we assert this...? can't compare NaNs... + // nocommit + assert !Float.isNaN(scores[slot]); } @Override @@ -857,6 +988,24 @@ // sorts descending: return second.compareTo(first); } + + @Override + public int compareDocToValue(int doc, Float valueObj) throws IOException { + final float value = valueObj.floatValue(); + float docValue = scorer.score(); + // TODO: should we assert this...? can't compare NaNs... + // nocommit + assert !Float.isNaN(docValue); + if (docValue < value) { + // reverse of FloatComparator + return 1; + } else if (docValue > value) { + // reverse of FloatComparator + return -1; + } else { + return 0; + } + } } /** Sorts by ascending docID */ @@ -904,6 +1053,19 @@ public Integer value(int slot) { return Integer.valueOf(docIDs[slot]); } + + @Override + public int compareDocToValue(int doc, Integer valueObj) { + final int value = valueObj.intValue(); + int docValue = docBase + doc; + if (docValue < value) { + return -1; + } else if (docValue > value) { + return 1; + } else { + return 0; + } + } } /** Sorts by field's natural Term sort order, using @@ -998,6 +1160,20 @@ throw new UnsupportedOperationException(); } + @Override + public int compareDocToValue(int doc, BytesRef value) { + BytesRef docValue = termsIndex.getTerm(doc, tempBR); + if (docValue == null) { + if (value == null) { + return 0; + } + return -1; + } else if (value == null) { + return 1; + } + return docValue.compareTo(value); + } + /** Base class for specialized (per bit width of the * ords) per-segment comparator. NOTE: this is messy; * we do this only because hotspot can't reliably inline @@ -1038,6 +1214,11 @@ } return val1.compareTo(val2); } + + @Override + public int compareDocToValue(int doc, BytesRef value) { + return TermOrdValComparator.this.compareDocToValue(doc, value); + } } // Used per-segment when bit width of doc->ord is 8: @@ -1385,6 +1566,11 @@ throw new UnsupportedOperationException(); } + @Override + public int compareDocToValue(int doc, BytesRef value) { + return termsIndex.getBytes(doc, tempBR).compareTo(value); + } + // TODO: would be nice to share these specialized impls // w/ TermOrdValComparator @@ -1422,6 +1608,11 @@ assert val2 != null; return comp.compare(val1, val2); } + + @Override + public int compareDocToValue(int doc, BytesRef value) { + return TermOrdValDocValuesComparator.this.compareDocToValue(doc, value); + } } // Used per-segment when bit width of doc->ord is 8: @@ -1801,6 +1992,11 @@ } return val1.compareTo(val2); } + + @Override + public int compareDocToValue(int doc, BytesRef value) { + return docTerms.getTerm(doc, tempBR).compareTo(value); + } } /** Sorts by field's natural Term sort order. All @@ -1869,6 +2065,11 @@ assert val2 != null; return val1.compareTo(val2); } + + @Override + public int compareDocToValue(int doc, BytesRef value) { + return docTerms.getBytes(doc, tempBR).compareTo(value); + } } final protected static int binarySearch(BytesRef br, DocTermsIndex a, BytesRef key) { Index: lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1337911) +++ lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -311,9 +311,45 @@ */ public TopFieldDocs search(Query query, Filter filter, int n, Sort sort) throws IOException { - return search(createNormalizedWeight(wrapFilter(query, filter)), n, sort); + return search(createNormalizedWeight(wrapFilter(query, filter)), n, sort, false, false); } + /** Search implementation with arbitrary sorting, plus + * control over whether document scores and max score + * should be computed. Finds + * the top n hits for query, applying + * filter if non-null, and sorting the hits by the criteria in + * sort. + * + *

NOTE: this does not compute scores by default; use + * {@link IndexSearcher#setDefaultFieldSortScoring} to + * enable scoring. + * + * @throws BooleanQuery.TooManyClauses + */ + public TopFieldDocs search(Query query, Filter filter, int n, + Sort sort, boolean doDocScores, boolean doMaxScore) throws IOException { + return search(createNormalizedWeight(wrapFilter(query, filter)), n, sort, doDocScores, doMaxScore); + } + + /** Finds the top n + * hits for query, applying filter if non-null, + * where all results are after a previous result (after). + *

+ * By passing the bottom result from a previous page as after, + * this method can be used for efficient 'deep-paging' across potentially + * large result sets. + * + * @throws BooleanQuery.TooManyClauses + */ + public TopDocs searchAfter(ScoreDoc after, Query query, Filter filter, int n, Sort sort) throws IOException { + if (after != null && !(after instanceof FieldDoc)) { + // nocommit remove if we fix type safety of TopFieldDocs... + throw new IllegalArgumentException("after must be a FieldDoc; got " + after); + } + return search(createNormalizedWeight(wrapFilter(query, filter)), (FieldDoc) after, n, sort, true, false, false); + } + /** * Search implementation with arbitrary sorting and no filter. * @param query The query to search for @@ -324,9 +360,47 @@ */ public TopFieldDocs search(Query query, int n, Sort sort) throws IOException { - return search(createNormalizedWeight(query), n, sort); + return search(createNormalizedWeight(query), n, sort, false, false); } + /** Finds the top n + * hits for query where all results are after a previous + * result (after). + *

+ * By passing the bottom result from a previous page as after, + * this method can be used for efficient 'deep-paging' across potentially + * large result sets. + * + * @throws BooleanQuery.TooManyClauses + */ + public TopDocs searchAfter(ScoreDoc after, Query query, int n, Sort sort) throws IOException { + if (after != null && !(after instanceof FieldDoc)) { + // nocommit remove if we fix type safety of TopFieldDocs... + throw new IllegalArgumentException("after must be a FieldDoc; got " + after); + } + return search(createNormalizedWeight(query), (FieldDoc) after, n, sort, true, false, false); + } + + /** Finds the top n + * hits for query where all results are after a previous + * result (after). + *

+ * By passing the bottom result from a previous page as after, + * this method can be used for efficient 'deep-paging' across potentially + * large result sets. + * + * @throws BooleanQuery.TooManyClauses + */ + public TopDocs searchAfter(ScoreDoc after, Query query, Filter filter, int n, Sort sort, + boolean doDocScores, boolean doMaxScore) throws IOException { + if (after != null && !(after instanceof FieldDoc)) { + // nocommit remove if we fix type safety of TopFieldDocs... + throw new IllegalArgumentException("after must be a FieldDoc; got " + after); + } + return search(createNormalizedWeight(wrapFilter(query, filter)), (FieldDoc) after, n, sort, true, + doDocScores, doMaxScore); + } + /** Expert: Low-level search implementation. Finds the top n * hits for query, applying filter if non-null. * @@ -393,8 +467,9 @@ * @throws BooleanQuery.TooManyClauses */ protected TopFieldDocs search(Weight weight, - final int nDocs, Sort sort) throws IOException { - return search(weight, nDocs, sort, true); + final int nDocs, Sort sort, + boolean doDocScores, boolean doMaxScore) throws IOException { + return search(weight, null, nDocs, sort, true, doDocScores, doMaxScore); } /** @@ -408,27 +483,29 @@ * then pass that to {@link #search(AtomicReaderContext[], Weight, * Collector)}.

*/ - protected TopFieldDocs search(Weight weight, int nDocs, - Sort sort, boolean fillFields) + protected TopFieldDocs search(Weight weight, FieldDoc after, int nDocs, + Sort sort, boolean fillFields, + boolean doDocScores, boolean doMaxScore) throws IOException { if (sort == null) throw new NullPointerException(); if (executor == null) { // use all leaves here! - return search (leafContexts, weight, nDocs, sort, fillFields); + return search(leafContexts, weight, after, nDocs, sort, fillFields, doDocScores, doMaxScore); } else { final TopFieldCollector topCollector = TopFieldCollector.create(sort, nDocs, + after, fillFields, - fieldSortDoTrackScores, - fieldSortDoMaxScore, + doDocScores, + doMaxScore, false); final Lock lock = new ReentrantLock(); final ExecutionHelper runner = new ExecutionHelper(executor); for (int i = 0; i < leafSlices.length; i++) { // search each leaf slice runner.submit( - new SearcherCallableWithSort(lock, this, leafSlices[i], weight, nDocs, topCollector, sort)); + new SearcherCallableWithSort(lock, this, leafSlices[i], weight, after, nDocs, topCollector, sort, doDocScores, doMaxScore)); } int totalHits = 0; float maxScore = Float.NEGATIVE_INFINITY; @@ -457,8 +534,8 @@ * then pass that to {@link #search(AtomicReaderContext[], Weight, * Collector)}.

*/ - protected TopFieldDocs search(AtomicReaderContext[] leaves, Weight weight, int nDocs, - Sort sort, boolean fillFields) throws IOException { + protected TopFieldDocs search(AtomicReaderContext[] leaves, Weight weight, FieldDoc after, int nDocs, + Sort sort, boolean fillFields, boolean doDocScores, boolean doMaxScore) throws IOException { // single thread int limit = reader.maxDoc(); if (limit == 0) { @@ -466,8 +543,9 @@ } nDocs = Math.min(nDocs, limit); - TopFieldCollector collector = TopFieldCollector.create(sort, nDocs, - fillFields, fieldSortDoTrackScores, fieldSortDoMaxScore, !weight.scoresDocsOutOfOrder()); + TopFieldCollector collector = TopFieldCollector.create(sort, nDocs, after, + fillFields, doDocScores, + doMaxScore, !weight.scoresDocsOutOfOrder()); search(leaves, weight, collector); return (TopFieldDocs) collector.topDocs(); } @@ -553,26 +631,6 @@ return weight.explain(leafContexts[n], deBasedDoc); } - private boolean fieldSortDoTrackScores; - private boolean fieldSortDoMaxScore; - - /** By default, no scores are computed when sorting by - * field (using {@link #search(Query,Filter,int,Sort)}). - * You can change that, per IndexSearcher instance, by - * calling this method. Note that this will incur a CPU - * cost. - * - * @param doTrackScores If true, then scores are - * returned for every matching document in {@link - * TopFieldDocs}. - * - * @param doMaxScore If true, then the max score for all - * matching docs is computed. */ - public void setDefaultFieldSortScoring(boolean doTrackScores, boolean doMaxScore) { - fieldSortDoTrackScores = doTrackScores; - fieldSortDoMaxScore = doMaxScore; - } - /** * Creates a normalized weight for a top-level {@link Query}. * The query is rewritten by this method and {@link Query#createWeight} called, @@ -626,7 +684,7 @@ } public TopDocs call() throws IOException { - final TopDocs docs = searcher.search (slice.leaves, weight, after, nDocs); + final TopDocs docs = searcher.search(slice.leaves, weight, after, nDocs); final ScoreDoc[] scoreDocs = docs.scoreDocs; //it would be so nice if we had a thread-safe insert lock.lock(); @@ -657,9 +715,13 @@ private final TopFieldCollector hq; private final Sort sort; private final LeafSlice slice; + private final FieldDoc after; + private final boolean doDocScores; + private final boolean doMaxScore; public SearcherCallableWithSort(Lock lock, IndexSearcher searcher, LeafSlice slice, Weight weight, - int nDocs, TopFieldCollector hq, Sort sort) { + FieldDoc after, int nDocs, TopFieldCollector hq, Sort sort, + boolean doDocScores, boolean doMaxScore) { this.lock = lock; this.searcher = searcher; this.weight = weight; @@ -667,6 +729,9 @@ this.hq = hq; this.sort = sort; this.slice = slice; + this.after = after; + this.doDocScores = doDocScores; + this.doMaxScore = doMaxScore; } private final class FakeScorer extends Scorer { @@ -707,8 +772,9 @@ public TopFieldDocs call() throws IOException { assert slice.leaves.length == 1; - final TopFieldDocs docs = searcher.search (slice.leaves, weight, nDocs, sort, true); + final TopFieldDocs docs = searcher.search(slice.leaves, weight, after, nDocs, sort, true, doDocScores, doMaxScore); lock.lock(); + // nocommit maxScore not carried properly here? try { final int base = slice.leaves[0].docBase; hq.setNextReader(slice.leaves[0]); Index: lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java =================================================================== --- lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (revision 1337911) +++ lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (working copy) @@ -444,7 +444,6 @@ } final IndexSearcher s = newSearcher(r); - s.setDefaultFieldSortScoring(true, true); final IndexSearcher joinS = newSearcher(joinR); Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1337911) +++ lucene/CHANGES.txt (working copy) @@ -266,6 +266,11 @@ * LUCENE-3970: Rename Fields.getUniqueFieldCount -> .size() and Terms.getUniqueTermCount -> .size(). (Iulius Curt via Mike McCandless) +* LUCENE-3514: IndexSearcher.setDefaultFieldSortScoring was removed + and replaced with per-search control via new expert search methods + that take two booleans indicating whether hit scores and max + score should be computed. (Mike McCandless) + Changes in Runtime Behavior * LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you @@ -857,6 +862,10 @@ * LUCENE-4039: Add AddIndexesTask to benchmark, which uses IW.addIndexes. (Shai Erera) +* LUCENE-3514: Added IndexSearcher.searchAfter when Sort is used, + returning results after a specified FieldDoc for deep + paging. (Mike McCandless) + Optimizations * LUCENE-2588: Don't store unnecessary suffixes when writing the terms Index: lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (revision 1337911) +++ lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (working copy) @@ -118,4 +118,16 @@ return collator.compare(first, second); } } + + @Override + public int compareDocToValue(int doc, String value) { + final BytesRef br = currentDocTerms.getTerm(doc, tempBR); + final String docValue; + if (br == null) { + docValue = null; + } else { + docValue = br.utf8ToString(); + } + return compareValues(docValue, value); + } } Index: lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java (revision 1337911) +++ lucene/queries/src/java/org/apache/lucene/queries/function/ValueSource.java (working copy) @@ -184,5 +184,18 @@ public Double value(int slot) { return values[slot]; } + + @Override + public int compareDocToValue(int doc, Double valueObj) { + final double value = valueObj.doubleValue(); + final double docValue = docVals.doubleVal(doc); + if (docValue < value) { + return -1; + } else if (docValue > value) { + return -1; + } else { + return 0; + } + } } } Index: lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java =================================================================== --- lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (revision 1337911) +++ lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (working copy) @@ -132,7 +132,6 @@ reader = iw.getReader(); searcher = newSearcher(reader); - searcher.setDefaultFieldSortScoring(true, true); iw.close(); } Index: solr/core/src/java/org/apache/solr/schema/RandomSortField.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/RandomSortField.java (revision 1337911) +++ solr/core/src/java/org/apache/solr/schema/RandomSortField.java (working copy) @@ -138,6 +138,12 @@ public Integer value(int slot) { return values[slot]; } + + @Override + public int compareDocToValue(int doc, Integer valueObj) { + // values will be positive... no overflow possible. + return hash(doc+seed) - valueObj.intValue(); + } }; } }; Index: solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (revision 1337911) +++ solr/core/src/java/org/apache/solr/handler/component/QueryElevationComponent.java (working copy) @@ -560,9 +560,14 @@ public Integer value(int slot) { return values[slot]; } + + @Override + public int compareDocToValue(int doc, Integer valueObj) throws IOException { + final int value = valueObj.intValue(); + final int docValue = docVal(doc); + return docValue - value; // values will be small enough that there is no overflow concern + } }; } + } } -} - - Index: solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java =================================================================== --- solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java (revision 1337911) +++ solr/core/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java (working copy) @@ -121,6 +121,11 @@ return TermOrdValComparator_SML.createComparator(context.reader(), this); } + @Override + public int compareDocToValue(int doc, Comparable docValue) { + throw new UnsupportedOperationException(); + } + // Base class for specialized (per bit width of the // ords) per-segment comparator. NOTE: this is messy; // we do this only because hotspot can't reliably inline @@ -216,6 +221,20 @@ public BytesRef value(int slot) { return values==null ? parent.NULL_VAL : values[slot]; } + + @Override + public int compareDocToValue(int doc, BytesRef value) { + final BytesRef docValue = termsIndex.getTerm(doc, tempBR); + if (docValue == null) { + if (value == null) { + return 0; + } + return 1; + } else if (value == null) { + return -1; + } + return docValue.compareTo(value); + } } // Used per-segment when bit width of doc->ord is 8: