Index: lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntMultiDocValues.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntMultiDocValues.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/IntMultiDocValues.java (revision ) @@ -0,0 +1,80 @@ +package org.apache.lucene.queries.function.docvalues; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.queries.function.ValueSource; + +/** + * Created by peng on 31/01/14. + */ +public abstract class IntMultiDocValues extends IntDocValues { + + public IntMultiDocValues(ValueSource vs) { + super(vs); + } + + public void byteVal(int doc, byte [] vals) { + int[] result = new int[vals.length]; + intVal(doc, result); + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongMultiDocValues.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/LongMultiDocValues.java (revision ) @@ -0,0 +1,80 @@ +package org.apache.lucene.queries.function.docvalues; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.queries.function.ValueSource; + +/** + * Created by peng on 31/01/14. + */ +public abstract class LongMultiDocValues extends LongDocValues { + + public LongMultiDocValues(ValueSource vs) { + super(vs); + } + + public void byteVal(int doc, byte [] vals) { + long[] result = new long[vals.length]; + longVal(doc, result); + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatMultiDocValues.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/FloatMultiDocValues.java (revision ) @@ -0,0 +1,80 @@ +package org.apache.lucene.queries.function.docvalues; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.queries.function.ValueSource; + +/** + * Created by peng on 31/01/14. + */ +public abstract class FloatMultiDocValues extends FloatDocValues { + + public FloatMultiDocValues(ValueSource vs) { + super(vs); + } + + public void byteVal(int doc, byte [] vals) { + float[] result = new float[vals.length]; + floatVal(doc, result); + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleMultiDocValues.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DoubleMultiDocValues.java (revision ) @@ -0,0 +1,81 @@ +package org.apache.lucene.queries.function.docvalues; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.queries.function.ValueSource; + +/** + * Created by peng on 31/01/14. + */ +public abstract class DoubleMultiDocValues extends DoubleDocValues { + + public DoubleMultiDocValues(ValueSource vs) { + super(vs); + } + + public void byteVal(int doc, byte [] vals) { + double[] result = new double[vals.length]; + doubleVal(doc, result); + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexMultiDocValues.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/docvalues/DocTermsIndexMultiDocValues.java (revision ) @@ -0,0 +1,77 @@ +package org.apache.lucene.queries.function.docvalues; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValueSource; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.UnicodeUtil; + +import java.io.IOException; + +public abstract class DocTermsIndexMultiDocValues extends DocTermsIndexDocValues { + + protected final SortedSetDocValues multiTermsIndex; + + public DocTermsIndexMultiDocValues(ValueSource vs, + AtomicReaderContext context, String field) throws IOException { + super(vs, context, field); + try { + multiTermsIndex = FieldCache.DEFAULT.getDocTermOrds(context.reader(), field); + } catch (RuntimeException e) { + throw new DocTermsIndexException(field, e); + } + } + + //If vals has lower dimension than number of values in the field the later will be truncated; + //Make sure Parent ValueSource extends MultiValueSource + @Override + public void strVal(int doc, String [] vals) + { + multiTermsIndex.setDocument(doc); + + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ElementValueSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/ElementValueSource.java (revision ) @@ -0,0 +1,125 @@ +package org.apache.lucene.queries.function.valuesource; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; + +import java.io.IOException; +import java.util.Map; + +/** + * This function simply use a + */ +public class ElementValueSource extends ValueSource { + private final MultiValueSource delegate; + private final int n; + + public ElementValueSource(MultiValueSource delegate, int n) + { + if (n>=delegate.dimension()||n<0) throw new UnsupportedOperationException("element index must be between 0 (inclusive) and dimensionality of the vector (exclusive)"); + + this.delegate=delegate; + this.n=n; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + final FunctionValues dv = delegate.getValues(context, readerContext); + + return new FunctionValues() { + @Override + public String toString(int doc) { + return null; + } + + @Override + public byte byteVal(int doc) + { + byte[] results = new byte[delegate.dimension()]; + dv.byteVal(doc,results); + return results[n]; + } + + @Override + public short shortVal(int doc) + { + short[] results = new short[delegate.dimension()]; + dv.shortVal(doc, results); + return results[n]; + } + + @Override + public float floatVal(int doc) + { + float[] results = new float[delegate.dimension()]; + dv.floatVal(doc, results); + return results[n]; + } + + @Override + public int intVal(int doc) + { + int[] results = new int[delegate.dimension()]; + dv.intVal(doc, results); + return results[n]; + } + + @Override + public long longVal(int doc) + { + long[] results = new long[delegate.dimension()]; + dv.longVal(doc, results); + return results[n]; + } + + @Override + public double doubleVal(int doc) + { + double[] results = new double[delegate.dimension()]; + dv.doubleVal(doc, results); + return results[n]; + } + + public String strVal(int doc) + { + String[] results = new String[delegate.dimension()]; + dv.strVal(doc, results); + return results[n]; + } + }; + } + + @Override + public boolean equals(Object o) { + if (o.getClass() != ElementValueSource.class) return false; + ElementValueSource other = (ElementValueSource)o; + return (this.delegate.equals(other.delegate))&&(this.n==other.n); + } + + @Override + public int hashCode() { + return n * 31+delegate.hashCode(); + } + + @Override + public String description() { + return '(' + delegate.description() + ")["+n+']'; + } +} Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiIntFieldSource.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiIntFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiIntFieldSource.java (revision ) @@ -0,0 +1,156 @@ +package org.apache.lucene.queries.function.valuesource; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.IntMultiDocValues; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.mutable.MutableValue; +import org.apache.lucene.util.mutable.MutableValueInt; + +import java.io.IOException; +import java.util.Map; + +/** +* Created by peng on 05/02/14. +*/ +public class MultiIntFieldSource extends MultiValueFieldCacheSource { + + protected final FieldCache.IntParser parser; + + public MultiIntFieldSource(String field) { + this(field, null); + } + + public MultiIntFieldSource(String field, FieldCache.IntParser parser) { + super(field); + this.parser = parser; + } + + @Override + public String description() { + return "multiint(" + field + ')'; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + final SortedSetDocValues multiTermsIndex = cache.getDocTermOrds(readerContext.reader(), field); + + final Bits valid = cache.getDocsWithField(readerContext.reader(), field); + + return new IntMultiDocValues(this) { + private BytesRef spare = new BytesRef(); + + @Override + public int intVal(int doc) { + multiTermsIndex.setDocument(doc); + + long ord = multiTermsIndex.nextOrd(); + if (ord == SortedSetDocValues.NO_MORE_ORDS) return 0; + multiTermsIndex.lookupOrd(ord, spare); + + return parseSpare(); + } + + private int parseSpare() + { + if (parser == null) { + // Confusing: must delegate to wrapper (vs simply + // setting parser = + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so + // cache key includes + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER: + try { + return FieldCache.DEFAULT_INT_PARSER.parseInt(spare); + } catch (NumberFormatException ne) { + return FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(spare); + } + } + else return parser.parseInt(spare); + } + + @Override + public Object objectVal(int doc) { + return valid.get(doc) ? intVal(doc) : null; + } + + @Override + public boolean exists(int doc) { + return intVal(doc) != 0 || valid.get(doc); + } + + @Override + public ValueFiller getValueFiller() { + return new ValueFiller() { + private final MutableValueInt mval = new MutableValueInt(); + + @Override + public MutableValue getValue() { + return mval; + } + + @Override + public void fillValue(int doc) { + mval.value = intVal(doc); + mval.exists = mval.value != 0 || valid.get(doc); + } + }; + } + + @Override + public void intVal(int doc, int[] vals) { + multiTermsIndex.setDocument(doc); + + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiStrFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiStrFieldSource.java (revision ) @@ -0,0 +1,65 @@ +package org.apache.lucene.queries.function.valuesource; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.DocTermsIndexMultiDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +public class MultiStrFieldSource extends MultiValueFieldCacheSource { + + public MultiStrFieldSource(String field) { + this(field,DEFAULTDIM); + } + + public MultiStrFieldSource(String field, int dim) { + super(field, dim); + } + + @Override + public String description() { + return "multistr(" + field + ')'; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + return new DocTermsIndexMultiDocValues(this, readerContext, field) { + + @Override + protected String toTerm(String readableValue) { + return readableValue; + } + + @Override + public Object objectVal(int doc) { + return strVal(doc); + } + + @Override + public String toString(int doc) { + String[] vals = new String[dimension]; + this.strVal(doc, vals); + return description() + '=' + Arrays.toString(vals); + } + }; + } +} Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiLongFieldSource.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiLongFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiLongFieldSource.java (revision ) @@ -0,0 +1,156 @@ +package org.apache.lucene.queries.function.valuesource; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.LongMultiDocValues; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.mutable.MutableValue; +import org.apache.lucene.util.mutable.MutableValueLong; + +import java.io.IOException; +import java.util.Map; + +/** +* Created by peng on 05/02/14. +*/ +public class MultiLongFieldSource extends MultiValueFieldCacheSource { + + protected final FieldCache.LongParser parser; + + public MultiLongFieldSource(String field) { + this(field, null); + } + + public MultiLongFieldSource(String field, FieldCache.LongParser parser) { + super(field); + this.parser = parser; + } + + @Override + public String description() { + return "multilong(" + field + ')'; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + final SortedSetDocValues multiTermsIndex = cache.getDocTermOrds(readerContext.reader(), field); + + final Bits valid = cache.getDocsWithField(readerContext.reader(), field); + + return new LongMultiDocValues(this) { + private BytesRef spare = new BytesRef(); + + @Override + public long longVal(int doc) { + multiTermsIndex.setDocument(doc); + + long ord = multiTermsIndex.nextOrd(); + if (ord == SortedSetDocValues.NO_MORE_ORDS) return 0; + multiTermsIndex.lookupOrd(ord, spare); + + return parseSpare(); + } + + private long parseSpare() + { + if (parser == null) { + // Confusing: must delegate to wrapper (vs simply + // setting parser = + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so + // cache key includes + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER: + try { + return FieldCache.DEFAULT_LONG_PARSER.parseLong(spare); + } catch (NumberFormatException ne) { + return FieldCache.NUMERIC_UTILS_LONG_PARSER.parseLong(spare); + } + } + else return parser.parseLong(spare); + } + + @Override + public Object objectVal(int doc) { + return valid.get(doc) ? longVal(doc) : null; + } + + @Override + public boolean exists(int doc) { + return longVal(doc) != 0 || valid.get(doc); + } + + @Override + public ValueFiller getValueFiller() { + return new ValueFiller() { + private final MutableValueLong mval = new MutableValueLong(); + + @Override + public MutableValue getValue() { + return mval; + } + + @Override + public void fillValue(int doc) { + mval.value = longVal(doc); + mval.exists = mval.value != 0 || valid.get(doc); + } + }; + } + + @Override + public void longVal(int doc, long[] vals) { + multiTermsIndex.setDocument(doc); + + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiFloatFieldSource.java (revision ) @@ -0,0 +1,156 @@ +package org.apache.lucene.queries.function.valuesource; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.FloatMultiDocValues; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.mutable.MutableValue; +import org.apache.lucene.util.mutable.MutableValueFloat; + +import java.io.IOException; +import java.util.Map; + +/** +* Created by peng on 05/02/14. +*/ +public class MultiFloatFieldSource extends MultiValueFieldCacheSource { + + protected final FieldCache.FloatParser parser; + + public MultiFloatFieldSource(String field) { + this(field, null); + } + + public MultiFloatFieldSource(String field, FieldCache.FloatParser parser) { + super(field); + this.parser = parser; + } + + @Override + public String description() { + return "multifloat(" + field + ')'; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + final SortedSetDocValues multiTermsIndex = cache.getDocTermOrds(readerContext.reader(), field); + + final Bits valid = cache.getDocsWithField(readerContext.reader(), field); + + return new FloatMultiDocValues(this) { + private BytesRef spare = new BytesRef(); + + @Override + public float floatVal(int doc) { + multiTermsIndex.setDocument(doc); + + long ord = multiTermsIndex.nextOrd(); + if (ord == SortedSetDocValues.NO_MORE_ORDS) return 0; + multiTermsIndex.lookupOrd(ord, spare); + + return parseSpare(); + } + + private float parseSpare() + { + if (parser == null) { + // Confusing: must delegate to wrapper (vs simply + // setting parser = + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so + // cache key includes + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER: + try { + return FieldCache.DEFAULT_FLOAT_PARSER.parseFloat(spare); + } catch (NumberFormatException ne) { + return FieldCache.NUMERIC_UTILS_FLOAT_PARSER.parseFloat(spare); + } + } + else return parser.parseFloat(spare); + } + + @Override + public Object objectVal(int doc) { + return valid.get(doc) ? floatVal(doc) : null; + } + + @Override + public boolean exists(int doc) { + return floatVal(doc) != 0 || valid.get(doc); + } + + @Override + public ValueFiller getValueFiller() { + return new ValueFiller() { + private final MutableValueFloat mval = new MutableValueFloat(); + + @Override + public MutableValue getValue() { + return mval; + } + + @Override + public void fillValue(int doc) { + mval.value = floatVal(doc); + mval.exists = mval.value != 0 || valid.get(doc); + } + }; + } + + @Override + public void floatVal(int doc, float[] vals) { + multiTermsIndex.setDocument(doc); + + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiDoubleFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiDoubleFieldSource.java (revision ) @@ -0,0 +1,156 @@ +package org.apache.lucene.queries.function.valuesource; + +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.docvalues.DoubleMultiDocValues; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.mutable.MutableValue; +import org.apache.lucene.util.mutable.MutableValueDouble; + +import java.io.IOException; +import java.util.Map; + +/** +* Created by peng on 05/02/14. +*/ +public class MultiDoubleFieldSource extends MultiValueFieldCacheSource { + + protected final FieldCache.DoubleParser parser; + + public MultiDoubleFieldSource(String field) { + this(field, null); + } + + public MultiDoubleFieldSource(String field, FieldCache.DoubleParser parser) { + super(field); + this.parser = parser; + } + + @Override + public String description() { + return "multidouble(" + field + ')'; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException { + final SortedSetDocValues multiTermsIndex = cache.getDocTermOrds(readerContext.reader(), field); + + final Bits valid = cache.getDocsWithField(readerContext.reader(), field); + + return new DoubleMultiDocValues(this) { + private BytesRef spare = new BytesRef(); + + @Override + public double doubleVal(int doc) { + multiTermsIndex.setDocument(doc); + + long ord = multiTermsIndex.nextOrd(); + if (ord == SortedSetDocValues.NO_MORE_ORDS) return 0; + multiTermsIndex.lookupOrd(ord, spare); + + return parseSpare(); + } + + private double parseSpare() + { + if (parser == null) { + // Confusing: must delegate to wrapper (vs simply + // setting parser = + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER) so + // cache key includes + // DEFAULT_FLOAT_PARSER/NUMERIC_UTILS_FLOAT_PARSER: + try { + return FieldCache.DEFAULT_DOUBLE_PARSER.parseDouble(spare); + } catch (NumberFormatException ne) { + return FieldCache.NUMERIC_UTILS_DOUBLE_PARSER.parseDouble(spare); + } + } + else return parser.parseDouble(spare); + } + + @Override + public Object objectVal(int doc) { + return valid.get(doc) ? doubleVal(doc) : null; + } + + @Override + public boolean exists(int doc) { + return doubleVal(doc) != 0 || valid.get(doc); + } + + @Override + public ValueFiller getValueFiller() { + return new ValueFiller() { + private final MutableValueDouble mval = new MutableValueDouble(); + + @Override + public MutableValue getValue() { + return mval; + } + + @Override + public void fillValue(int doc) { + mval.value = doubleVal(doc); + mval.exists = mval.value != 0 || valid.get(doc); + } + }; + } + + @Override + public void doubleVal(int doc, double[] vals) { + multiTermsIndex.setDocument(doc); + + for (int i=0;iUTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueFieldCacheSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/MultiValueFieldCacheSource.java (revision ) @@ -0,0 +1,74 @@ +package org.apache.lucene.queries.function.valuesource; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.FieldCache; + +public abstract class MultiValueFieldCacheSource extends MultiValueSource { + + protected int dimension; + + protected static final int DEFAULTDIM=10;//TODO: dynamic dimensionality, needs support from more powerful FunctionValue + + protected final String field; + protected final FieldCache cache = FieldCache.DEFAULT; + + public MultiValueFieldCacheSource(String field) { + this(field, DEFAULTDIM); + } + + public MultiValueFieldCacheSource(String field, int dim) { + this.field=field; + this.dimension =dim; + } + + public FieldCache getFieldCache() { + return cache; + } + + public String getField() { + return field; + } + + @Override + public String description() { + return field; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof MultiValueFieldCacheSource)) return false; + MultiValueFieldCacheSource other = (MultiValueFieldCacheSource)o; + return this.field.equals(other.field) + && this.cache == other.cache; + } + + @Override + public int hashCode() { + return cache.hashCode() + field.hashCode(); + } + + @Override + public int dimension() { + return dimension; + } + + public void setDimension(int dimension) { + this.dimension = dimension; + } +} Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IntFieldSource.java (revision ) @@ -17,18 +17,16 @@ package org.apache.lucene.queries.function.valuesource; -import java.io.IOException; -import java.util.Map; - import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.queries.function.FunctionValues; -import org.apache.lucene.queries.function.ValueSourceScorer; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.FieldCache; import org.apache.lucene.util.Bits; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueInt; + +import java.io.IOException; +import java.util.Map; /** * Obtains int field values from {@link FieldCache#getInts} and makes those Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (revision ) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/FloatFieldSource.java (revision ) @@ -17,9 +17,6 @@ package org.apache.lucene.queries.function.valuesource; -import java.io.IOException; -import java.util.Map; - import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.FloatDocValues; @@ -27,6 +24,9 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueFloat; + +import java.io.IOException; +import java.util.Map; /** * Obtains float field values from {@link FieldCache#getFloats} and makes those Index: lucene/queries/src/test/org/apache/lucene/queries/function/TestMultiFieldCacheFunctions.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/src/test/org/apache/lucene/queries/function/TestMultiFieldCacheFunctions.java (revision ) +++ lucene/queries/src/test/org/apache/lucene/queries/function/TestMultiFieldCacheFunctions.java (revision ) @@ -0,0 +1,218 @@ +package org.apache.lucene.queries.function; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatField; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.queries.function.valuesource.ElementValueSource; +import org.apache.lucene.queries.function.valuesource.MultiDoubleFieldSource; +import org.apache.lucene.queries.function.valuesource.MultiFloatFieldSource; +import org.apache.lucene.queries.function.valuesource.MultiIntFieldSource; +import org.apache.lucene.queries.function.valuesource.MultiLongFieldSource; +import org.apache.lucene.queries.function.valuesource.MultiValueSource; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * Created by peng on 04/02/14. + */ +public class TestMultiFieldCacheFunctions extends LuceneTestCase { + static Directory dir; + static IndexReader reader; + static IndexSearcher searcher; + + static final List documents = Arrays.asList(new String[][]{ + /* id, byte, double, float, int, long, short, string, text */ + new String[]{"2.7", "2.718", "2.7", "2.718", "35", "4343", "945", "9450", "this is a test test test","test2 test2"}, + new String[]{"3.14", "3.14159", "3.14", "3.14159", "54", "1954", "123", "1230", "second test","second2 test"}, + }); + + @BeforeClass + public static void beforeClass() throws Exception { + dir = LuceneTestCase.newDirectory(); + IndexWriterConfig iwConfig = LuceneTestCase.newIndexWriterConfig(LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(LuceneTestCase.random())); + iwConfig.setMergePolicy(LuceneTestCase.newLogMergePolicy()); + RandomIndexWriter iw = new RandomIndexWriter(LuceneTestCase.random(), dir, iwConfig); + Document document = new Document(); + + Field doubleField = new DoubleField("double", 0.0, Field.Store.NO); + document.add(doubleField); + Field doubleField2 = new DoubleField("double", 0.0, Field.Store.NO); + document.add(doubleField2); + Field floatField = new FloatField("float", 0.0f, Field.Store.NO); + document.add(floatField); + Field floatField2 = new FloatField("float", 0.0f, Field.Store.NO); + document.add(floatField2); + Field intField = new IntField("int", 0, Field.Store.NO); + document.add(intField); + Field intField2 = new IntField("int", 0, Field.Store.NO); + document.add(intField2); + Field longField = new LongField("long", 0l, Field.Store.NO); + document.add(longField); + Field longField2 = new LongField("long", 0l, Field.Store.NO); + document.add(longField2); + Field stringField = new StringField("string", "", Field.Store.NO); + document.add(stringField); + Field stringField2 = new StringField("string", "", Field.Store.NO); + document.add(stringField2); + + for (String [] doc : documents) { + doubleField.setDoubleValue(Double.parseDouble(doc[0])); + doubleField2.setDoubleValue(Double.parseDouble(doc[1])); + floatField.setFloatValue(Float.parseFloat(doc[2])); + floatField2.setFloatValue(Float.parseFloat(doc[3])); + intField.setIntValue(Integer.parseInt(doc[4])); + intField2.setIntValue(Integer.parseInt(doc[5])); + longField.setLongValue(Long.parseLong(doc[6])); + longField2.setLongValue(Long.parseLong(doc[7])); + stringField.setStringValue(doc[8]); + stringField2.setStringValue(doc[9]); + iw.addDocument(document); + } + + reader = iw.getReader(); + searcher = LuceneTestCase.newSearcher(reader); + iw.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + searcher = null; + reader.close(); + reader = null; + dir.close(); + dir = null; + } + +// public void testMultiStr() throws IOException { +// ValueSource lit = new LiteralValueSource("second test"); +// ValueSource multiStr = new MultiStrFieldSource("string"); +// ValueSource dist = new MultiStringDistanceFunction(lit, multiStr, new LevensteinDistance()); +// +// FunctionQuery fq = new FunctionQuery(dist); +// +// TopDocs topDocs = searcher.search(fq, 2); +// System.out.println(topDocs); //TODO +// } + + @Test + public void testMultiDouble() throws IOException { + MultiValueSource multiDouble = new MultiDoubleFieldSource("double"); +// List l = new ArrayList(); +// l.add(new ConstValueSource(0.5f)); +// l.add(new ConstValueSource(0.6f)); +// MultiValueSource ref = new VectorValueSource(l); + ValueSource f0 = new ElementValueSource(multiDouble,0); + ValueSource f1 = new ElementValueSource(multiDouble,1); + + FunctionQuery fq0 = new FunctionQuery(f0); + FunctionQuery fq1 = new FunctionQuery(f1); + + TopDocs topDocs = searcher.search(fq0, 2); + assertEquals(topDocs.scoreDocs[0].score,3.14,0.0001f); + assertEquals(topDocs.scoreDocs[1].score,2.7,0.0001f); + + topDocs = searcher.search(fq1, 2); + assertEquals(topDocs.scoreDocs[0].score,3.14159,0.0001f); + assertEquals(topDocs.scoreDocs[1].score, 2.718, 0.0001f); + } + + @Test + public void testMultiFloat() throws IOException { + MultiValueSource multiFloat = new MultiFloatFieldSource("float"); +// List l = new ArrayList(); +// l.add(new ConstValueSource(0.5f)); +// l.add(new ConstValueSource(0.6f)); +// MultiValueSource ref = new VectorValueSource(l); + ValueSource f0 = new ElementValueSource(multiFloat,0); + ValueSource f1 = new ElementValueSource(multiFloat,1); + + FunctionQuery fq0 = new FunctionQuery(f0); + FunctionQuery fq1 = new FunctionQuery(f1); + + TopDocs topDocs = searcher.search(fq0, 2); + assertEquals(topDocs.scoreDocs[0].score,3.14,0.0001f); + assertEquals(topDocs.scoreDocs[1].score,2.7,0.0001f); + + topDocs = searcher.search(fq1, 2); + assertEquals(topDocs.scoreDocs[0].score,3.14159,0.0001f); + assertEquals(topDocs.scoreDocs[1].score, 2.718, 0.0001f); + } + + @Test + public void testMultiInt() throws IOException { + MultiValueSource multiInt = new MultiIntFieldSource("int"); +// List l = new ArrayList(); +// l.add(new ConstValueSource(0.5f)); +// l.add(new ConstValueSource(0.6f)); +// MultiValueSource ref = new VectorValueSource(l); + ValueSource f0 = new ElementValueSource(multiInt,0); + ValueSource f1 = new ElementValueSource(multiInt,1); + + FunctionQuery fq0 = new FunctionQuery(f0); + FunctionQuery fq1 = new FunctionQuery(f1); + + TopDocs topDocs = searcher.search(fq0, 2); + assertEquals(topDocs.scoreDocs[0].score,54,0.0001f); + assertEquals(topDocs.scoreDocs[1].score,35,0.0001f); + + topDocs = searcher.search(fq1, 2); + assertEquals(topDocs.scoreDocs[0].score,4343,0.0001f); + assertEquals(topDocs.scoreDocs[1].score, 1954, 0.0001f); + } + + @Test + public void testMultiLong() throws IOException { + MultiValueSource multiLong = new MultiLongFieldSource("long"); +// List l = new ArrayList(); +// l.add(new ConstValueSource(0.5f)); +// l.add(new ConstValueSource(0.6f)); +// MultiValueSource ref = new VectorValueSource(l); + ValueSource f0 = new ElementValueSource(multiLong,0); + ValueSource f1 = new ElementValueSource(multiLong,1); + + FunctionQuery fq0 = new FunctionQuery(f0); + FunctionQuery fq1 = new FunctionQuery(f1); + + TopDocs topDocs = searcher.search(fq0, 2); + assertEquals(topDocs.scoreDocs[0].score,945,0.0001f); + assertEquals(topDocs.scoreDocs[1].score,123,0.0001f); + + topDocs = searcher.search(fq1, 2); + assertEquals(topDocs.scoreDocs[0].score,9450,0.0001f); + assertEquals(topDocs.scoreDocs[1].score, 1230, 0.0001f); + } +} Index: lucene/queries/queries.iml IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- lucene/queries/queries.iml (revision ) +++ lucene/queries/queries.iml (revision ) @@ -14,6 +14,7 @@ +