Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 803620) +++ CHANGES.txt (working copy) @@ -668,6 +668,14 @@ 35. LUCENE-1790: Added BoostingFunctionTermQuery to enable scoring of payloads based on the maximum payload seen for a document. Slight refactoring of Similarity and other payload queries (Grant Ingersoll) + +36. LUCENE-1789: Added utility class + oal.search.function.MultiValueSource to ease the transition to + segment based searching for any apps that directly call + oal.search.function.* APIs. This class wraps any other + ValueSource, but takes care when composite (multi-segment) are + passed to not double RAM usage in the FieldCache. (Chris + Hostetter, Mark Miller, Mike McCandless) Optimizations Index: src/test/org/apache/lucene/search/function/TestOrdValues.java =================================================================== --- src/test/org/apache/lucene/search/function/TestOrdValues.java (revision 803620) +++ src/test/org/apache/lucene/search/function/TestOrdValues.java (working copy) @@ -57,9 +57,9 @@ IndexSearcher s = new IndexSearcher(dir); ValueSource vs; if (inOrder) { - vs = new OrdFieldSource(field); + vs = new MultiValueSource(new OrdFieldSource(field)); } else { - vs = new ReverseOrdFieldSource(field); + vs = new MultiValueSource(new ReverseOrdFieldSource(field)); } Query q = new ValueSourceQuery(vs); Index: src/test/org/apache/lucene/search/function/TestValueSource.java =================================================================== --- src/test/org/apache/lucene/search/function/TestValueSource.java (revision 0) +++ src/test/org/apache/lucene/search/function/TestValueSource.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.search.function; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.*; +import org.apache.lucene.store.*; +import org.apache.lucene.search.*; +import org.apache.lucene.search.function.*; +import org.apache.lucene.analysis.*; +import org.apache.lucene.index.*; +import org.apache.lucene.document.*; + +public class TestValueSource extends LuceneTestCase { + + public void testMultiValueSource() throws Exception { + Directory dir = new MockRAMDirectory(); + IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.UNLIMITED); + Document doc = new Document(); + Field f = new Field("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED); + doc.add(f); + + for(int i=0;i<17;i++) { + f.setValue(""+i); + w.addDocument(doc); + w.commit(); + } + + IndexReader r = w.getReader(); + w.close(); + + assertTrue(r.getSequentialSubReaders().length > 1); + + ValueSource s1 = new IntFieldSource("field"); + DocValues v1 = s1.getValues(r); + DocValues v2 = new MultiValueSource(s1).getValues(r); + + for(int i=0;iNOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class IntFieldSource extends FieldCacheSource { private FieldCache.IntParser parser; Index: src/java/org/apache/lucene/search/function/ShortFieldSource.java =================================================================== --- src/java/org/apache/lucene/search/function/ShortFieldSource.java (revision 803620) +++ src/java/org/apache/lucene/search/function/ShortFieldSource.java (working copy) @@ -36,6 +36,16 @@ * * @see org.apache.lucene.search.function.FieldCacheSource for requirements * on the field. + * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class ShortFieldSource extends FieldCacheSource { private FieldCache.ShortParser parser; Index: src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java =================================================================== --- src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (revision 803620) +++ src/java/org/apache/lucene/search/function/ReverseOrdFieldSource.java (working copy) @@ -45,6 +45,15 @@ * The APIs introduced here might change in the future and will not be * supported anymore in such a case. * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class ReverseOrdFieldSource extends ValueSource { Index: src/java/org/apache/lucene/search/function/ByteFieldSource.java =================================================================== --- src/java/org/apache/lucene/search/function/ByteFieldSource.java (revision 803620) +++ src/java/org/apache/lucene/search/function/ByteFieldSource.java (working copy) @@ -36,6 +36,16 @@ * * @see org.apache.lucene.search.function.FieldCacheSource for requirements * on the field. + * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class ByteFieldSource extends FieldCacheSource { private FieldCache.ByteParser parser; Index: src/java/org/apache/lucene/search/function/FieldCacheSource.java =================================================================== --- src/java/org/apache/lucene/search/function/FieldCacheSource.java (revision 803620) +++ src/java/org/apache/lucene/search/function/FieldCacheSource.java (working copy) @@ -39,6 +39,15 @@ * The APIs introduced here might change in the future and will not be * supported anymore in such a case. * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public abstract class FieldCacheSource extends ValueSource { private String field; Index: src/java/org/apache/lucene/search/function/MultiValueSource.java =================================================================== --- src/java/org/apache/lucene/search/function/MultiValueSource.java (revision 0) +++ src/java/org/apache/lucene/search/function/MultiValueSource.java (revision 0) @@ -0,0 +1,124 @@ +package org.apache.lucene.search.function; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.ReaderUtil; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Explanation; + +/** This class wraps another ValueSource, but protects + * against accidental double RAM usage in FieldCache when + * a composite reader is passed to {@link #getValues}. + * + *

NOTE: this class adds a CPU penalty to every + * lookup, as it must resolve the incoming document to the + * right sub-reader using a binary search.

+ * + * @deprecated This class is temporary, to ease the + * migration to segment-based searching. Please change your + * code to not pass composite readers to these APIs. */ + +public final class MultiValueSource extends ValueSource { + + final ValueSource other; + public MultiValueSource(ValueSource other) { + this.other = other; + } + + public DocValues getValues(IndexReader reader) throws IOException { + + IndexReader[] subReaders = reader.getSequentialSubReaders(); + if (subReaders != null) { + // This is a composite reader + return new MultiDocValues(subReaders); + } else { + // Already an atomic reader -- just delegate + return other.getValues(reader); + } + } + + public String description() { + return other.description(); + } + + public boolean equals(Object o) { + if (o instanceof MultiValueSource) { + return ((MultiValueSource) o).other.equals(other); + } else { + return false; + } + } + + public int hashCode() { + return 31 * other.hashCode(); + } + + private final class MultiDocValues extends DocValues { + + final DocValues[] docValues; + final int[] docStarts; + + MultiDocValues(IndexReader[] subReaders) throws IOException { + docValues = new DocValues[subReaders.length]; + docStarts = new int[subReaders.length]; + int base = 0; + for(int i=0;i * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class OrdFieldSource extends ValueSource { Index: src/java/org/apache/lucene/search/function/FloatFieldSource.java =================================================================== --- src/java/org/apache/lucene/search/function/FloatFieldSource.java (revision 803620) +++ src/java/org/apache/lucene/search/function/FloatFieldSource.java (working copy) @@ -37,6 +37,15 @@ * @see org.apache.lucene.search.function.FieldCacheSource for requirements * on the field. * + *

NOTE: with the switch in 2.9 to segment-based + * searching, if {@link #getValues} is invoked with a + * composite (multi-segment) reader, this can easily cause + * double RAM usage for the values in the FieldCache. It's + * best to switch your application to pass only atomic + * (single segment) readers to this API. Alternatively, for + * a short-term fix, you could wrap your ValueSource using + * {@link MultiValueSource}, which costs more CPU per lookup + * but will not consume double the FieldCache RAM.

*/ public class FloatFieldSource extends FieldCacheSource { private FieldCache.FloatParser parser;