diff -r fed9c167c8aa lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java --- a/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java Wed Sep 01 08:04:39 2010 -0400 @@ -51,6 +51,7 @@ "org.apache.lucene.analysis.standard.StandardAnalyzer")); String defaultField = config.get("file.query.maker.default.field", DocMaker.BODY_FIELD); QueryParser qp = new QueryParser(Version.LUCENE_CURRENT, defaultField, anlzr); + qp.setAllowLeadingWildcard(true); List qq = new ArrayList(); String fileName = config.get("file.query.maker.file", null); diff -r fed9c167c8aa lucene/contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java --- a/lucene/contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java Wed Sep 01 08:04:39 2010 -0400 @@ -192,12 +192,13 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { docValues = FieldCache.DEFAULT.getInts(reader, "parser", new FieldCache.IntParser() { public final int parseInt(BytesRef termRef) { return (termRef.utf8ToString().charAt(0)-'A') * 123456; } }); + return this; } @Override diff -r fed9c167c8aa lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/DistanceFieldComparatorSource.java --- a/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/DistanceFieldComparatorSource.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/DistanceFieldComparatorSource.java Wed Sep 01 08:04:39 2010 -0400 @@ -109,12 +109,14 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { // each reader in a segmented base // has an offset based on the maxDocs of previous readers offset = docBase; + + return this; } @Override diff -r fed9c167c8aa lucene/src/java/org/apache/lucene/search/FieldComparator.java --- a/lucene/src/java/org/apache/lucene/search/FieldComparator.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/src/java/org/apache/lucene/search/FieldComparator.java Wed Sep 01 08:04:39 2010 -0400 @@ -31,6 +31,9 @@ import org.apache.lucene.search.FieldCache.DocTermsIndex; import org.apache.lucene.search.FieldCache.DocTerms; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.packed.Direct8; +import org.apache.lucene.util.packed.Direct16; +import org.apache.lucene.util.packed.Direct32; import org.apache.lucene.util.packed.PackedInts; /** @@ -138,14 +141,19 @@ public abstract void copy(int slot, int doc) throws IOException; /** - * Set a new Reader. All doc correspond to the current Reader. + * Set a new Reader. All subsequent docIDs are relative to + * the current reader (you must add docBase if you need to + * map it to a top-level docID). * * @param reader current reader * @param docBase docBase of this reader + * @return the comparator to use for this segment; most + * comparators can just return "this" to reuse the same + * comparator across segments * @throws IOException * @throws IOException */ - public abstract void setNextReader(IndexReader reader, int docBase) throws IOException; + public abstract FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException; /** Sets the Scorer to use in case a document's score is * needed. @@ -196,8 +204,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getBytes(reader, field, parser); + return this; } @Override @@ -239,11 +248,12 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) { + public FieldComparator setNextReader(IndexReader reader, int docBase) { // TODO: can we "map" our docIDs to the current // reader? saves having to then subtract on every // compare call this.docBase = docBase; + return this; } @Override @@ -303,8 +313,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getDoubles(reader, field, parser); + return this; } @Override @@ -368,8 +379,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getFloats(reader, field, parser); + return this; } @Override @@ -437,8 +449,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser); + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { + currentReaderValues = FieldCache.DEFAULT.getInts(reader, field, parser); + return this; } @Override @@ -502,8 +515,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getLongs(reader, field, parser); + return this; } @Override @@ -551,7 +565,8 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) { + public FieldComparator setNextReader(IndexReader reader, int docBase) { + return this; } @Override @@ -603,8 +618,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentReaderValues = FieldCache.DEFAULT.getShorts(reader, field, parser); + return this; } @Override @@ -679,8 +695,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { currentDocTerms = FieldCache.DEFAULT.getTerms(reader, field); + return this; } @Override @@ -710,7 +727,6 @@ private final BytesRef[] values; private final int[] readerGen; - private PackedInts.Reader currentDocToOrd; private int currentReaderGen = -1; private DocTermsIndex termsIndex; private final String field; @@ -731,10 +747,7 @@ @Override public int compare(int slot1, int slot2) { if (readerGen[slot1] == readerGen[slot2]) { - int cmp = ords[slot1] - ords[slot2]; - if (cmp != 0) { - return cmp; - } + return ords[slot1] - ords[slot2]; } final BytesRef val1 = values[slot1]; @@ -752,60 +765,307 @@ @Override public int compareBottom(int doc) { - assert bottomSlot != -1; - if (bottomSameReader) { - // ord is precisely comparable, even in the equal case - return bottomOrd - (int) currentDocToOrd.get(doc); - } else { - // ord is only approx comparable: if they are not - // equal, we can use that; if they are equal, we - // must fallback to compare by value - final int order = (int) currentDocToOrd.get(doc); - final int cmp = bottomOrd - order; - if (cmp != 0) { - return cmp; + throw new UnsupportedOperationException(); + } + + @Override + public void copy(int slot, int doc) { + throw new UnsupportedOperationException(); + } + + // Base class for specialized (per bit width of the + // ords) per-segment comparator. NOTE: this is messy; + // we do this only because hotspot can't reliably inline + // the underlying array access when looking up doc->ord + private abstract class PerSegmentComparator extends FieldComparator { + + @Override + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { + return TermOrdValComparator.this.setNextReader(reader, docBase); + } + + @Override + public int compare(int slot1, int slot2) { + return TermOrdValComparator.this.compare(slot1, slot2); + } + + @Override + public void setBottom(final int bottom) { + TermOrdValComparator.this.setBottom(bottom); + } + + @Override + public Comparable value(int slot) { + return TermOrdValComparator.this.value(slot); + } + } + + // Used per-segment when bit width of doc->ord is 8: + private final class ByteOrdComparator extends PerSegmentComparator { + private final byte[] readerOrds; + private final DocTermsIndex termsIndex; + private final int docBase; + + public ByteOrdComparator(byte[] readerOrds, DocTermsIndex termsIndex, int docBase) { + this.readerOrds = readerOrds; + this.termsIndex = termsIndex; + this.docBase = docBase; + } + + @Override + public int compareBottom(int doc) { + assert bottomSlot != -1; + if (bottomSameReader) { + // ord is precisely comparable, even in the equal case + return bottomOrd - (readerOrds[doc]&0xFF); + } else { + // ord is only approx comparable: if they are not + // equal, we can use that; if they are equal, we + // must fallback to compare by value + final int order = readerOrds[doc]&0xFF; + final int cmp = bottomOrd - order; + if (cmp != 0) { + return cmp; + } + + if (bottomValue == null) { + if (order == 0) { + // unset + return 0; + } + // bottom wins + return -1; + } else if (order == 0) { + // doc wins + return 1; + } + termsIndex.lookup(order, tempBR); + return bottomValue.compareTo(tempBR); } + } - if (bottomValue == null) { - if (order == 0) { - // unset - return 0; + @Override + public void copy(int slot, int doc) { + final int ord = readerOrds[doc]&0xFF; + if (ord == 0) { + values[slot] = null; + } else { + ords[slot] = ord; + assert ord > 0; + if (values[slot] == null) { + values[slot] = new BytesRef(); } - // bottom wins - return -1; - } else if (order == 0) { - // doc wins - return 1; + termsIndex.lookup(ord, values[slot]); } - termsIndex.lookup(order, tempBR); - return bottomValue.compareTo(tempBR); + readerGen[slot] = currentReaderGen; + } + } + + // Used per-segment when bit width of doc->ord is 16: + private final class ShortOrdComparator extends PerSegmentComparator { + private final short[] readerOrds; + private final DocTermsIndex termsIndex; + private final int docBase; + + public ShortOrdComparator(short[] readerOrds, DocTermsIndex termsIndex, int docBase) { + this.readerOrds = readerOrds; + this.termsIndex = termsIndex; + this.docBase = docBase; + } + + @Override + public int compareBottom(int doc) { + assert bottomSlot != -1; + if (bottomSameReader) { + // ord is precisely comparable, even in the equal case + return bottomOrd - (readerOrds[doc]&0xFFFF); + } else { + // ord is only approx comparable: if they are not + // equal, we can use that; if they are equal, we + // must fallback to compare by value + final int order = readerOrds[doc]&0xFFFF; + final int cmp = bottomOrd - order; + if (cmp != 0) { + return cmp; + } + + if (bottomValue == null) { + if (order == 0) { + // unset + return 0; + } + // bottom wins + return -1; + } else if (order == 0) { + // doc wins + return 1; + } + termsIndex.lookup(order, tempBR); + return bottomValue.compareTo(tempBR); + } + } + + @Override + public void copy(int slot, int doc) { + final int ord = readerOrds[doc]&0xFFFF; + if (ord == 0) { + values[slot] = null; + } else { + ords[slot] = ord; + assert ord > 0; + if (values[slot] == null) { + values[slot] = new BytesRef(); + } + termsIndex.lookup(ord, values[slot]); + } + readerGen[slot] = currentReaderGen; + } + } + + // Used per-segment when bit width of doc->ord is 32: + private final class IntOrdComparator extends PerSegmentComparator { + private final int[] readerOrds; + private final DocTermsIndex termsIndex; + private final int docBase; + + public IntOrdComparator(int[] readerOrds, DocTermsIndex termsIndex, int docBase) { + this.readerOrds = readerOrds; + this.termsIndex = termsIndex; + this.docBase = docBase; + } + + @Override + public int compareBottom(int doc) { + assert bottomSlot != -1; + if (bottomSameReader) { + // ord is precisely comparable, even in the equal case + return bottomOrd - readerOrds[doc]; + } else { + // ord is only approx comparable: if they are not + // equal, we can use that; if they are equal, we + // must fallback to compare by value + final int order = readerOrds[doc]; + final int cmp = bottomOrd - order; + if (cmp != 0) { + return cmp; + } + + if (bottomValue == null) { + if (order == 0) { + // unset + return 0; + } + // bottom wins + return -1; + } else if (order == 0) { + // doc wins + return 1; + } + termsIndex.lookup(order, tempBR); + return bottomValue.compareTo(tempBR); + } + } + + @Override + public void copy(int slot, int doc) { + final int ord = readerOrds[doc]; + if (ord == 0) { + values[slot] = null; + } else { + ords[slot] = ord; + assert ord > 0; + if (values[slot] == null) { + values[slot] = new BytesRef(); + } + termsIndex.lookup(ord, values[slot]); + } + readerGen[slot] = currentReaderGen; + } + } + + // Used per-segment when bit width is not a native array + // size (8, 16, 32): + private final class AnyOrdComparator extends PerSegmentComparator { + private final PackedInts.Reader readerOrds; + private final DocTermsIndex termsIndex; + private final int docBase; + + public AnyOrdComparator(PackedInts.Reader readerOrds, DocTermsIndex termsIndex, int docBase) { + this.readerOrds = readerOrds; + this.termsIndex = termsIndex; + this.docBase = docBase; + } + + @Override + public int compareBottom(int doc) { + assert bottomSlot != -1; + if (bottomSameReader) { + // ord is precisely comparable, even in the equal case + return bottomOrd - (int) readerOrds.get(doc); + } else { + // ord is only approx comparable: if they are not + // equal, we can use that; if they are equal, we + // must fallback to compare by value + final int order = (int) readerOrds.get(doc); + final int cmp = bottomOrd - order; + if (cmp != 0) { + return cmp; + } + + if (bottomValue == null) { + if (order == 0) { + // unset + return 0; + } + // bottom wins + return -1; + } else if (order == 0) { + // doc wins + return 1; + } + termsIndex.lookup(order, tempBR); + return bottomValue.compareTo(tempBR); + } + } + + @Override + public void copy(int slot, int doc) { + final int ord = (int) readerOrds.get(doc); + if (ord == 0) { + values[slot] = null; + } else { + ords[slot] = ord; + assert ord > 0; + if (values[slot] == null) { + values[slot] = new BytesRef(); + } + termsIndex.lookup(ord, values[slot]); + } + readerGen[slot] = currentReaderGen; } } @Override - public void copy(int slot, int doc) { - final int ord = (int) currentDocToOrd.get(doc); - if (ord == 0) { - values[slot] = null; + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { + termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field); + final PackedInts.Reader docToOrd = termsIndex.getDocToOrd(); + FieldComparator perSegComp; + if (docToOrd instanceof Direct8) { + perSegComp = new ByteOrdComparator(((Direct8) docToOrd).getArray(), termsIndex, docBase); + } else if (docToOrd instanceof Direct16) { + perSegComp = new ShortOrdComparator(((Direct16) docToOrd).getArray(), termsIndex, docBase); + } else if (docToOrd instanceof Direct32) { + perSegComp = new IntOrdComparator(((Direct32) docToOrd).getArray(), termsIndex, docBase); } else { - ords[slot] = ord; - assert ord >= 0; - if (values[slot] == null) { - values[slot] = new BytesRef(); - } - termsIndex.lookup(ord, values[slot]); + perSegComp = new AnyOrdComparator(docToOrd, termsIndex, docBase); } - readerGen[slot] = currentReaderGen; - } - @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { - termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field); - currentDocToOrd = termsIndex.getDocToOrd(); currentReaderGen++; if (bottomSlot != -1) { - setBottom(bottomSlot); + perSegComp.setBottom(bottomSlot); } + + return perSegComp; } @Override @@ -813,36 +1073,37 @@ bottomSlot = bottom; bottomValue = values[bottomSlot]; - if (bottomValue == null) { - // 0 ord is null for all segments - assert ords[bottomSlot] == 0; - bottomOrd = 0; + if (currentReaderGen == readerGen[bottomSlot]) { + bottomOrd = ords[bottomSlot]; bottomSameReader = true; } else { - final int index = binarySearch(tempBR, termsIndex, bottomValue); - if (index < 0) { - bottomOrd = -index - 2; - bottomSameReader = false; + if (bottomValue == null) { + // 0 ord is null for all segments + assert ords[bottomSlot] == 0; + bottomOrd = 0; + bottomSameReader = true; + readerGen[bottomSlot] = currentReaderGen; } else { - bottomOrd = index; - // exact value match - bottomSameReader = true; + final int index = binarySearch(tempBR, termsIndex, bottomValue); + if (index < 0) { + bottomOrd = -index - 2; + bottomSameReader = false; + } else { + bottomOrd = index; + // exact value match + bottomSameReader = true; + } } } + if (bottomSameReader) { + readerGen[bottomSlot] = currentReaderGen; + } } @Override public Comparable value(int slot) { return values[slot]; } - - public int getBottomSlot() { - return bottomSlot; - } - - public String getField() { - return field; - } } /** Sorts by field's natural Term sort order. All @@ -901,8 +1162,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { docTerms = FieldCache.DEFAULT.getTerms(reader, field); + return this; } @Override diff -r fed9c167c8aa lucene/src/java/org/apache/lucene/search/TopFieldCollector.java --- a/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/src/java/org/apache/lucene/search/TopFieldCollector.java Wed Sep 01 08:04:39 2010 -0400 @@ -46,7 +46,7 @@ private static class OneComparatorNonScoringCollector extends TopFieldCollector { - final FieldComparator comparator; + FieldComparator comparator; final int reverseMul; public OneComparatorNonScoringCollector(FieldValueHitQueue queue, @@ -92,7 +92,7 @@ @Override public void setNextReader(IndexReader reader, int docBase) throws IOException { this.docBase = docBase; - comparator.setNextReader(reader, docBase); + comparator = comparator.setNextReader(reader, docBase); } @Override @@ -446,7 +446,7 @@ public void setNextReader(IndexReader reader, int docBase) throws IOException { this.docBase = docBase; for (int i = 0; i < comparators.length; i++) { - comparators[i].setNextReader(reader, docBase); + comparators[i] = comparators[i].setNextReader(reader, docBase); } } diff -r fed9c167c8aa lucene/src/test/org/apache/lucene/search/JustCompileSearch.java --- a/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Wed Sep 01 08:04:39 2010 -0400 @@ -239,7 +239,7 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } diff -r fed9c167c8aa lucene/src/test/org/apache/lucene/search/TestElevationComparator.java --- a/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/src/test/org/apache/lucene/search/TestElevationComparator.java Wed Sep 01 08:04:39 2010 -0400 @@ -175,8 +175,9 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { idIndex = FieldCache.DEFAULT.getTermsIndex(reader, fieldname); + return this; } @Override diff -r fed9c167c8aa lucene/src/test/org/apache/lucene/search/TestSort.java --- a/lucene/src/test/org/apache/lucene/search/TestSort.java Tue Aug 31 19:50:12 2010 +0000 +++ b/lucene/src/test/org/apache/lucene/search/TestSort.java Wed Sep 01 08:04:39 2010 -0400 @@ -459,12 +459,13 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { docValues = FieldCache.DEFAULT.getInts(reader, "parser", new FieldCache.IntParser() { public final int parseInt(final BytesRef term) { return (term.bytes[term.offset]-'A') * 123456; } }); + return this; } @Override diff -r fed9c167c8aa solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java --- a/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Tue Aug 31 19:50:12 2010 +0000 +++ b/solr/src/java/org/apache/solr/handler/component/QueryElevationComponent.java Wed Sep 01 08:04:39 2010 -0400 @@ -493,8 +493,9 @@ values[slot] = docVal(doc); } - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { idIndex = FieldCache.DEFAULT.getTermsIndex(reader, fieldname); + return this; } public Comparable value(int slot) { diff -r fed9c167c8aa solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java --- a/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java Tue Aug 31 19:50:12 2010 +0000 +++ b/solr/src/java/org/apache/solr/search/MissingStringLastComparatorSource.java Wed Sep 01 08:04:39 2010 -0400 @@ -166,7 +166,7 @@ } @Override - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { termsIndex = FieldCache.DEFAULT.getTermsIndex(reader, field); currentReaderGen++; assert termsIndex.numOrd() > 0; @@ -174,6 +174,7 @@ convert(bottomSlot); bottomOrd = ords[bottomSlot]; } + return this; } @Override diff -r fed9c167c8aa solr/src/java/org/apache/solr/search/function/ValueSource.java --- a/solr/src/java/org/apache/solr/search/function/ValueSource.java Tue Aug 31 19:50:12 2010 +0000 +++ b/solr/src/java/org/apache/solr/search/function/ValueSource.java Wed Sep 01 08:04:39 2010 -0400 @@ -157,8 +157,9 @@ values[slot] = docVals.doubleVal(doc); } - public void setNextReader(IndexReader reader, int docBase) throws IOException { + public FieldComparator setNextReader(IndexReader reader, int docBase) throws IOException { docVals = getValues(Collections.emptyMap(), reader); + return this; } public void setBottom(final int bottom) {