- * In order to take advantage of this, you must override - * {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)} - * which returns 1 by default. - *
- * Payload scores are aggregated using a pluggable {@link PayloadFunction}. - **/ -public class BoostingFunctionTermQuery extends SpanTermQuery implements PayloadQuery{ - protected PayloadFunction function; - private boolean includeSpanScore; - - public BoostingFunctionTermQuery(Term term, PayloadFunction function) { - this(term, function, true); - } - - public BoostingFunctionTermQuery(Term term, PayloadFunction function, boolean includeSpanScore) { - super(term); - this.function = function; - this.includeSpanScore = includeSpanScore; - } - - - - public Weight createWeight(Searcher searcher) throws IOException { - return new BoostingFunctionTermWeight(this, searcher); - } - - protected class BoostingFunctionTermWeight extends SpanWeight { - - public BoostingFunctionTermWeight(BoostingFunctionTermQuery query, Searcher searcher) throws IOException { - super(query, searcher); - } - - public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - return new BoostingFunctionSpanScorer((TermSpans) query.getSpans(reader), this, - similarity, reader.norms(query.getField())); - } - - protected class BoostingFunctionSpanScorer extends SpanScorer { - //TODO: is this the best way to allocate this? - protected byte[] payload = new byte[256]; - protected TermPositions positions; - protected float payloadScore; - protected int payloadsSeen; - - public BoostingFunctionSpanScorer(TermSpans spans, Weight weight, Similarity similarity, - byte[] norms) throws IOException { - super(spans, weight, similarity, norms); - positions = spans.getPositions(); - } - - protected boolean setFreqCurrentDoc() throws IOException { - if (!more) { - return false; - } - doc = spans.doc(); - freq = 0.0f; - payloadScore = 0; - payloadsSeen = 0; - Similarity similarity1 = getSimilarity(); - while (more && doc == spans.doc()) { - int matchLength = spans.end() - spans.start(); - - freq += similarity1.sloppyFreq(matchLength); - processPayload(similarity1); - - more = spans.next();//this moves positions to the next match in this document - } - return more || (freq != 0); - } - - - protected void processPayload(Similarity similarity) throws IOException { - if (positions.isPayloadAvailable()) { - payload = positions.getPayload(payload, 0); - payloadScore = function.currentScore(doc, term.field(), spans.start(), spans.end(), payloadsSeen, payloadScore, - similarity.scorePayload(doc, term.field(), spans.start(), spans.end(), payload, 0, positions.getPayloadLength())); - payloadsSeen++; - - } else { - //zero out the payload? - } - } - - /** - * - * @return {@link #getSpanScore()} * {@link #getPayloadScore()} - * @throws IOException - */ - public float score() throws IOException { - - return includeSpanScore ? getSpanScore() * getPayloadScore() : getPayloadScore(); - } - - /** - * Returns the SpanScorer score only. - *
- * Should not be overriden without good cause! - * - * @return the score for just the Span part w/o the payload - * @throws IOException - * - * @see #score() - */ - protected float getSpanScore() throws IOException{ - return super.score(); - } - - /** - * The score for the payload - * @return The score, as calculated by {@link PayloadFunction#docScore(int, String, int, float)} - */ - protected float getPayloadScore() { - return function.docScore(doc, term.field(), payloadsSeen, payloadScore); - } - - - public Explanation explain(final int doc) throws IOException { - ComplexExplanation result = new ComplexExplanation(); - Explanation nonPayloadExpl = super.explain(doc); - result.addDetail(nonPayloadExpl); - //QUESTION: Is there a way to avoid this skipTo call? We need to know whether to load the payload or not - Explanation payloadBoost = new Explanation(); - result.addDetail(payloadBoost); - - - float payloadScore = getPayloadScore(); - payloadBoost.setValue(payloadScore); - //GSI: I suppose we could toString the payload, but I don't think that would be a good idea - payloadBoost.setDescription("scorePayload(...)"); - result.setValue(nonPayloadExpl.getValue() * payloadScore); - result.setDescription("btq, product of:"); - result.setMatch(nonPayloadExpl.getValue()==0 ? Boolean.FALSE : Boolean.TRUE); // LUCENE-1303 - return result; - } - - } - } - - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((function == null) ? 0 : function.hashCode()); - result = prime * result + (includeSpanScore ? 1231 : 1237); - return result; - } - - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - BoostingFunctionTermQuery other = (BoostingFunctionTermQuery) obj; - if (function == null) { - if (other.function != null) - return false; - } else if (!function.equals(other.function)) - return false; - if (includeSpanScore != other.includeSpanScore) - return false; - return true; - } - - -} Index: src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (revision 806233) +++ src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (working copy) @@ -4,20 +4,28 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermPositions; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.Similarity; import org.apache.lucene.search.Weight; +import org.apache.lucene.search.spans.SpanScorer; +import org.apache.lucene.search.spans.SpanTermQuery; +import org.apache.lucene.search.spans.SpanWeight; import org.apache.lucene.search.spans.TermSpans; /** - * Copyright 2004 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,52 +34,187 @@ */ /** - * The BoostingTermQuery is very similar to the {@link org.apache.lucene.search.spans.SpanTermQuery} except - * that it factors in the value of the payload located at each of the positions where the + * This class is very similar to + * {@link org.apache.lucene.search.spans.SpanTermQuery} except that it factors + * in the value of the payload located at each of the positions where the * {@link org.apache.lucene.index.Term} occurs. *- * In order to take advantage of this, you must override {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)} + * In order to take advantage of this, you must override + * {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)} * which returns 1 by default. *
- * Payload scores are averaged across term occurrences in the document. - * - * @see org.apache.lucene.search.Similarity#scorePayload(String, byte[], int, int) - * - * @deprecated See {@link org.apache.lucene.search.payloads.BoostingFunctionTermQuery} - */ -public class BoostingTermQuery extends BoostingFunctionTermQuery implements PayloadQuery{ + * Payload scores are aggregated using a pluggable {@link PayloadFunction}. + **/ +public class BoostingTermQuery extends SpanTermQuery implements PayloadQuery { + protected PayloadFunction function; + private boolean includeSpanScore; public BoostingTermQuery(Term term) { - this(term, true); + this(term, new AveragePayloadFunction(), true); } public BoostingTermQuery(Term term, boolean includeSpanScore) { - super(term, new AveragePayloadFunction(), includeSpanScore); + this(term, new AveragePayloadFunction(), includeSpanScore); + } + + public BoostingTermQuery(Term term, PayloadFunction function) { + this(term, function, true); + } + + public BoostingTermQuery(Term term, PayloadFunction function, + boolean includeSpanScore) { + super(term); + this.function = function; + this.includeSpanScore = includeSpanScore; } public Weight createWeight(Searcher searcher) throws IOException { return new BoostingTermWeight(this, searcher); } - protected class BoostingTermWeight extends BoostingFunctionTermWeight { + protected class BoostingTermWeight extends SpanWeight { - public BoostingTermWeight(BoostingTermQuery query, Searcher searcher) throws IOException { + public BoostingTermWeight(BoostingTermQuery query, Searcher searcher) + throws IOException { super(query, searcher); } - public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { - return new BoostingFunctionSpanScorer((TermSpans) query.getSpans(reader), this, - similarity, reader.norms(query.getField())); + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, + boolean topScorer) throws IOException { + return new BoostingFunctionSpanScorer((TermSpans) query.getSpans(reader), + this, similarity, reader.norms(query.getField())); } + protected class BoostingFunctionSpanScorer extends SpanScorer { + // TODO: is this the best way to allocate this? + protected byte[] payload = new byte[256]; + protected TermPositions positions; + protected float payloadScore; + protected int payloadsSeen; + + public BoostingFunctionSpanScorer(TermSpans spans, Weight weight, + Similarity similarity, byte[] norms) throws IOException { + super(spans, weight, similarity, norms); + positions = spans.getPositions(); + } + + protected boolean setFreqCurrentDoc() throws IOException { + if (!more) { + return false; + } + doc = spans.doc(); + freq = 0.0f; + payloadScore = 0; + payloadsSeen = 0; + Similarity similarity1 = getSimilarity(); + while (more && doc == spans.doc()) { + int matchLength = spans.end() - spans.start(); + + freq += similarity1.sloppyFreq(matchLength); + processPayload(similarity1); + + more = spans.next();// this moves positions to the next match in this + // document + } + return more || (freq != 0); + } + + protected void processPayload(Similarity similarity) throws IOException { + if (positions.isPayloadAvailable()) { + payload = positions.getPayload(payload, 0); + payloadScore = function.currentScore(doc, term.field(), + spans.start(), spans.end(), payloadsSeen, payloadScore, + similarity.scorePayload(doc, term.field(), spans.start(), spans + .end(), payload, 0, positions.getPayloadLength())); + payloadsSeen++; + + } else { + // zero out the payload? + } + } + + /** + * + * @return {@link #getSpanScore()} * {@link #getPayloadScore()} + * @throws IOException + */ + public float score() throws IOException { + + return includeSpanScore ? getSpanScore() * getPayloadScore() + : getPayloadScore(); + } + + /** + * Returns the SpanScorer score only. + *
+ * Should not be overriden without good cause! + * + * @return the score for just the Span part w/o the payload + * @throws IOException + * + * @see #score() + */ + protected float getSpanScore() throws IOException { + return super.score(); + } + + /** + * The score for the payload + * + * @return The score, as calculated by + * {@link PayloadFunction#docScore(int, String, int, float)} + */ + protected float getPayloadScore() { + return function.docScore(doc, term.field(), payloadsSeen, payloadScore); + } + + public Explanation explain(final int doc) throws IOException { + ComplexExplanation result = new ComplexExplanation(); + Explanation nonPayloadExpl = super.explain(doc); + result.addDetail(nonPayloadExpl); + // QUESTION: Is there a way to avoid this skipTo call? We need to know + // whether to load the payload or not + Explanation payloadBoost = new Explanation(); + result.addDetail(payloadBoost); + + float payloadScore = getPayloadScore(); + payloadBoost.setValue(payloadScore); + // GSI: I suppose we could toString the payload, but I don't think that + // would be a good idea + payloadBoost.setDescription("scorePayload(...)"); + result.setValue(nonPayloadExpl.getValue() * payloadScore); + result.setDescription("btq, product of:"); + result.setMatch(nonPayloadExpl.getValue() == 0 ? Boolean.FALSE + : Boolean.TRUE); // LUCENE-1303 + return result; + } + + } } + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((function == null) ? 0 : function.hashCode()); + result = prime * result + (includeSpanScore ? 1231 : 1237); + return result; + } - public boolean equals(Object o) { - if (!(o instanceof BoostingTermQuery)) + public boolean equals(Object obj) { + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) return false; - BoostingTermQuery other = (BoostingTermQuery) o; - return (this.getBoost() == other.getBoost()) - && this.term.equals(other.term); + BoostingTermQuery other = (BoostingTermQuery) obj; + if (function == null) { + if (other.function != null) + return false; + } else if (!function.equals(other.function)) + return false; + if (includeSpanScore != other.includeSpanScore) + return false; + return true; } } Index: src/test/org/apache/lucene/search/payloads/BoostingFunctionTermQueryTest.java =================================================================== --- src/test/org/apache/lucene/search/payloads/BoostingFunctionTermQueryTest.java (revision 806233) +++ src/test/org/apache/lucene/search/payloads/BoostingFunctionTermQueryTest.java (working copy) @@ -1,322 +0,0 @@ -package org.apache.lucene.search.payloads; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.English; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.QueryUtils; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.CheckHits; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.spans.SpanTermQuery; -import org.apache.lucene.search.spans.Spans; -import org.apache.lucene.search.spans.TermSpans; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.LowerCaseTokenizer; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; - -import java.io.Reader; -import java.io.IOException; - - -/** - * - * - **/ -public class BoostingFunctionTermQueryTest extends LuceneTestCase { - private IndexSearcher searcher; - private BoostingSimilarity similarity = new BoostingSimilarity(); - private byte[] payloadField = new byte[]{1}; - private byte[] payloadMultiField1 = new byte[]{2}; - private byte[] payloadMultiField2 = new byte[]{4}; - protected RAMDirectory directory; - - public BoostingFunctionTermQueryTest(String s) { - super(s); - } - - private class PayloadAnalyzer extends Analyzer { - - - public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream result = new LowerCaseTokenizer(reader); - result = new PayloadFilter(result, fieldName); - return result; - } - } - - private class PayloadFilter extends TokenFilter { - String fieldName; - int numSeen = 0; - - PayloadAttribute payloadAtt; - - public PayloadFilter(TokenStream input, String fieldName) { - super(input); - this.fieldName = fieldName; - payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); - } - - public boolean incrementToken() throws IOException { - boolean hasNext = input.incrementToken(); - if (hasNext) { - if (fieldName.equals("field")) { - payloadAtt.setPayload(new Payload(payloadField)); - } else if (fieldName.equals("multiField")) { - if (numSeen % 2 == 0) { - payloadAtt.setPayload(new Payload(payloadMultiField1)); - } else { - payloadAtt.setPayload(new Payload(payloadMultiField2)); - } - numSeen++; - } - return true; - } else { - return false; - } - } - } - - protected void setUp() throws Exception { - super.setUp(); - directory = new RAMDirectory(); - PayloadAnalyzer analyzer = new PayloadAnalyzer(); - IndexWriter writer - = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); - writer.setSimilarity(similarity); - //writer.infoStream = System.out; - for (int i = 0; i < 1000; i++) { - Document doc = new Document(); - Field noPayloadField = new Field(PayloadHelper.NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED); - //noPayloadField.setBoost(0); - doc.add(noPayloadField); - doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED)); - doc.add(new Field("multiField", English.intToEnglish(i) + " " + English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED)); - writer.addDocument(doc); - } - writer.optimize(); - writer.close(); - - searcher = new IndexSearcher(directory, true); - searcher.setSimilarity(similarity); - } - - public void test() throws IOException { - BoostingFunctionTermQuery query = new BoostingFunctionTermQuery(new Term("field", "seventy"), - new MaxPayloadFunction()); - TopDocs hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); - assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); - - //they should all have the exact same score, because they all contain seventy once, and we set - //all the other similarity factors to be 1 - - assertTrue(hits.getMaxScore() + " does not equal: " + 1, hits.getMaxScore() == 1); - for (int i = 0; i < hits.scoreDocs.length; i++) { - ScoreDoc doc = hits.scoreDocs[i]; - assertTrue(doc.score + " does not equal: " + 1, doc.score == 1); - } - CheckHits.checkExplanations(query, PayloadHelper.FIELD, searcher, true); - Spans spans = query.getSpans(searcher.getIndexReader()); - assertTrue("spans is null and it shouldn't be", spans != null); - assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); - /*float score = hits.score(0); - for (int i =1; i < hits.length(); i++) - { - assertTrue("scores are not equal and they should be", score == hits.score(i)); - }*/ - - } - - public void testQuery() { - BoostingFunctionTermQuery boostingFuncTermQuery = new BoostingFunctionTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), - new MaxPayloadFunction()); - QueryUtils.check(boostingFuncTermQuery); - - SpanTermQuery spanTermQuery = new SpanTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy")); - - assertTrue(boostingFuncTermQuery.equals(spanTermQuery) == spanTermQuery.equals(boostingFuncTermQuery)); - - BoostingFunctionTermQuery boostingFuncTermQuery2 = new BoostingFunctionTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), - new AveragePayloadFunction()); - - QueryUtils.checkUnequal(boostingFuncTermQuery, boostingFuncTermQuery2); - } - - public void testMultipleMatchesPerDoc() throws Exception { - BoostingFunctionTermQuery query = new BoostingFunctionTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), - new MaxPayloadFunction()); - TopDocs hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); - assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); - - //they should all have the exact same score, because they all contain seventy once, and we set - //all the other similarity factors to be 1 - - //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); - assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); - //there should be exactly 10 items that score a 4, all the rest should score a 2 - //The 10 items are: 70 + i*100 where i in [0-9] - int numTens = 0; - for (int i = 0; i < hits.scoreDocs.length; i++) { - ScoreDoc doc = hits.scoreDocs[i]; - if (doc.doc % 10 == 0) { - numTens++; - assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); - } else { - assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); - } - } - assertTrue(numTens + " does not equal: " + 10, numTens == 10); - CheckHits.checkExplanations(query, "field", searcher, true); - Spans spans = query.getSpans(searcher.getIndexReader()); - assertTrue("spans is null and it shouldn't be", spans != null); - assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); - //should be two matches per document - int count = 0; - //100 hits times 2 matches per hit, we should have 200 in count - while (spans.next()) { - count++; - } - assertTrue(count + " does not equal: " + 200, count == 200); - } - - //Set includeSpanScore to false, in which case just the payload score comes through. - public void testIgnoreSpanScorer() throws Exception { - BoostingFunctionTermQuery query = new BoostingFunctionTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), - new MaxPayloadFunction(), false); - - IndexSearcher theSearcher = new IndexSearcher(directory, true); - theSearcher.setSimilarity(new FullSimilarity()); - TopDocs hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); - assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); - - //they should all have the exact same score, because they all contain seventy once, and we set - //all the other similarity factors to be 1 - - //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); - assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); - //there should be exactly 10 items that score a 4, all the rest should score a 2 - //The 10 items are: 70 + i*100 where i in [0-9] - int numTens = 0; - for (int i = 0; i < hits.scoreDocs.length; i++) { - ScoreDoc doc = hits.scoreDocs[i]; - if (doc.doc % 10 == 0) { - numTens++; - assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); - } else { - assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); - } - } - assertTrue(numTens + " does not equal: " + 10, numTens == 10); - CheckHits.checkExplanations(query, "field", searcher, true); - Spans spans = query.getSpans(searcher.getIndexReader()); - assertTrue("spans is null and it shouldn't be", spans != null); - assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); - //should be two matches per document - int count = 0; - //100 hits times 2 matches per hit, we should have 200 in count - while (spans.next()) { - count++; - } - } - - public void testNoMatch() throws Exception { - BoostingFunctionTermQuery query = new BoostingFunctionTermQuery(new Term(PayloadHelper.FIELD, "junk"), - new MaxPayloadFunction()); - TopDocs hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); - assertTrue("hits Size: " + hits.totalHits + " is not: " + 0, hits.totalHits == 0); - - } - - public void testNoPayload() throws Exception { - BoostingFunctionTermQuery q1 = new BoostingFunctionTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero"), - new MaxPayloadFunction()); - BoostingFunctionTermQuery q2 = new BoostingFunctionTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo"), - new MaxPayloadFunction()); - BooleanClause c1 = new BooleanClause(q1, BooleanClause.Occur.MUST); - BooleanClause c2 = new BooleanClause(q2, BooleanClause.Occur.MUST_NOT); - BooleanQuery query = new BooleanQuery(); - query.add(c1); - query.add(c2); - TopDocs hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); - assertTrue("hits Size: " + hits.totalHits + " is not: " + 1, hits.totalHits == 1); - int[] results = new int[1]; - results[0] = 0;//hits.scoreDocs[0].doc; - CheckHits.checkHitCollector(query, PayloadHelper.NO_PAYLOAD_FIELD, searcher, results); - } - - // must be static for weight serialization tests - static class BoostingSimilarity extends DefaultSimilarity { - - // TODO: Remove warning after API has been finalized - public float scorePayload(int docId, String fieldName, int start, int end, byte[] payload, int offset, int length) { - //we know it is size 4 here, so ignore the offset/length - return payload[0]; - } - - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - //Make everything else 1 so we see the effect of the payload - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - public float lengthNorm(String fieldName, int numTerms) { - return 1; - } - - public float queryNorm(float sumOfSquaredWeights) { - return 1; - } - - public float sloppyFreq(int distance) { - return 1; - } - - public float coord(int overlap, int maxOverlap) { - return 1; - } - - public float idf(int docFreq, int numDocs) { - return 1; - } - - public float tf(float freq) { - return freq == 0 ? 0 : 1; - } - } - - static class FullSimilarity extends DefaultSimilarity{ - public float scorePayload(int docId, String fieldName, byte[] payload, int offset, int length) { - //we know it is size 4 here, so ignore the offset/length - return payload[0]; - } - } - -} Index: src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java =================================================================== --- src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (revision 806233) +++ src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (working copy) @@ -1,11 +1,11 @@ package org.apache.lucene.search.payloads; - /** - * Copyright 2004 The Apache Software Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -34,20 +34,28 @@ import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; import org.apache.lucene.search.spans.TermSpans; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.English; import org.apache.lucene.util.LuceneTestCase; + +/** + * + * + **/ public class TestBoostingTermQuery extends LuceneTestCase { private IndexSearcher searcher; private BoostingSimilarity similarity = new BoostingSimilarity(); private byte[] payloadField = new byte[]{1}; private byte[] payloadMultiField1 = new byte[]{2}; private byte[] payloadMultiField2 = new byte[]{4}; + protected RAMDirectory directory; public TestBoostingTermQuery(String s) { super(s); @@ -97,7 +105,7 @@ protected void setUp() throws Exception { super.setUp(); - RAMDirectory directory = new RAMDirectory(); + directory = new RAMDirectory(); PayloadAnalyzer analyzer = new PayloadAnalyzer(); IndexWriter writer = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); @@ -115,12 +123,13 @@ writer.optimize(); writer.close(); - searcher = new IndexSearcher(directory); + searcher = new IndexSearcher(directory, true); searcher.setSimilarity(similarity); } public void test() throws IOException { - BoostingTermQuery query = new BoostingTermQuery(new Term("field", "seventy")); + BoostingTermQuery query = new BoostingTermQuery(new Term("field", "seventy"), + new MaxPayloadFunction()); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); @@ -144,8 +153,23 @@ }*/ } + + public void testQuery() { + BoostingTermQuery boostingFuncTermQuery = new BoostingTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), + new MaxPayloadFunction()); + QueryUtils.check(boostingFuncTermQuery); + + SpanTermQuery spanTermQuery = new SpanTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy")); - public void testMultipleMatchesPerDoc() throws Exception { + assertTrue(boostingFuncTermQuery.equals(spanTermQuery) == spanTermQuery.equals(boostingFuncTermQuery)); + + BoostingTermQuery boostingFuncTermQuery2 = new BoostingTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), + new AveragePayloadFunction()); + + QueryUtils.checkUnequal(boostingFuncTermQuery, boostingFuncTermQuery2); + } + + public void testMultipleMatchesPerDocAvgFunction() throws Exception { BoostingTermQuery query = new BoostingTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy")); TopDocs hits = searcher.search(query, null, 100); assertTrue("hits is null and it shouldn't be", hits != null); @@ -182,6 +206,85 @@ assertTrue(count + " does not equal: " + 200, count == 200); } + public void testMultipleMatchesPerDocMaxFunction() throws Exception { + BoostingTermQuery query = new BoostingTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), + new MaxPayloadFunction()); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); + + //they should all have the exact same score, because they all contain seventy once, and we set + //all the other similarity factors to be 1 + + //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); + assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); + //there should be exactly 10 items that score a 4, all the rest should score a 2 + //The 10 items are: 70 + i*100 where i in [0-9] + int numTens = 0; + for (int i = 0; i < hits.scoreDocs.length; i++) { + ScoreDoc doc = hits.scoreDocs[i]; + if (doc.doc % 10 == 0) { + numTens++; + assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); + } else { + assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); + } + } + assertTrue(numTens + " does not equal: " + 10, numTens == 10); + CheckHits.checkExplanations(query, "field", searcher, true); + Spans spans = query.getSpans(searcher.getIndexReader()); + assertTrue("spans is null and it shouldn't be", spans != null); + assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); + //should be two matches per document + int count = 0; + //100 hits times 2 matches per hit, we should have 200 in count + while (spans.next()) { + count++; + } + assertTrue(count + " does not equal: " + 200, count == 200); + } + + //Set includeSpanScore to false, in which case just the payload score comes through. + public void testIgnoreSpanScorer() throws Exception { + BoostingTermQuery query = new BoostingTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy"), + new MaxPayloadFunction(), false); + + IndexSearcher theSearcher = new IndexSearcher(directory, true); + theSearcher.setSimilarity(new FullSimilarity()); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); + + //they should all have the exact same score, because they all contain seventy once, and we set + //all the other similarity factors to be 1 + + //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); + assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); + //there should be exactly 10 items that score a 4, all the rest should score a 2 + //The 10 items are: 70 + i*100 where i in [0-9] + int numTens = 0; + for (int i = 0; i < hits.scoreDocs.length; i++) { + ScoreDoc doc = hits.scoreDocs[i]; + if (doc.doc % 10 == 0) { + numTens++; + assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); + } else { + assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); + } + } + assertTrue(numTens + " does not equal: " + 10, numTens == 10); + CheckHits.checkExplanations(query, "field", searcher, true); + Spans spans = query.getSpans(searcher.getIndexReader()); + assertTrue("spans is null and it shouldn't be", spans != null); + assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); + //should be two matches per document + int count = 0; + //100 hits times 2 matches per hit, we should have 200 in count + while (spans.next()) { + count++; + } + } + public void testNoMatch() throws Exception { BoostingTermQuery query = new BoostingTermQuery(new Term(PayloadHelper.FIELD, "junk")); TopDocs hits = searcher.search(query, null, 100); @@ -190,9 +293,20 @@ } + public void testNoMatchMaxFunction() throws Exception { + BoostingTermQuery query = new BoostingTermQuery(new Term(PayloadHelper.FIELD, "junk"), + new MaxPayloadFunction()); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 0, hits.totalHits == 0); + + } + public void testNoPayload() throws Exception { - BoostingTermQuery q1 = new BoostingTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero")); - BoostingTermQuery q2 = new BoostingTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo")); + BoostingTermQuery q1 = new BoostingTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero"), + new MaxPayloadFunction()); + BoostingTermQuery q2 = new BoostingTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo"), + new MaxPayloadFunction()); BooleanClause c1 = new BooleanClause(q1, BooleanClause.Occur.MUST); BooleanClause c2 = new BooleanClause(q2, BooleanClause.Occur.MUST_NOT); BooleanQuery query = new BooleanQuery(); @@ -206,15 +320,15 @@ CheckHits.checkHitCollector(query, PayloadHelper.NO_PAYLOAD_FIELD, searcher, results); } - // must be static for weight serialization tests + // must be static for weight serialization tests static class BoostingSimilarity extends DefaultSimilarity { + // TODO: Remove warning after API has been finalized public float scorePayload(int docId, String fieldName, int start, int end, byte[] payload, int offset, int length) { //we know it is size 4 here, so ignore the offset/length return payload[0]; } - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -242,4 +356,12 @@ return freq == 0 ? 0 : 1; } } + + static class FullSimilarity extends DefaultSimilarity{ + public float scorePayload(int docId, String fieldName, byte[] payload, int offset, int length) { + //we know it is size 4 here, so ignore the offset/length + return payload[0]; + } + } + }