Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 801868) +++ CHANGES.txt (working copy) @@ -664,7 +664,10 @@ disable loading them with a new constructor switch. (Mark Miller) 34. LUCENE-1341: Added BoostingNearQuery to enable SpanNearQuery functionality - with payloads (Peter Keegan, Grant Ingersoll) + with payloads (Peter Keegan, Grant Ingersoll) + +35. LUCENE-1790: Added BoostingMaxTermQuery to enable scoring of payloads + based on the maximum payload seen for a document. (Grant Ingersoll) Optimizations Index: src/test/org/apache/lucene/search/payloads/BoostingMaxTermQueryTest.java =================================================================== --- src/test/org/apache/lucene/search/payloads/BoostingMaxTermQueryTest.java (revision 0) +++ src/test/org/apache/lucene/search/payloads/BoostingMaxTermQueryTest.java (revision 0) @@ -0,0 +1,251 @@ +package org.apache.lucene.search.payloads; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.English; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.CheckHits; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.spans.Spans; +import org.apache.lucene.search.spans.TermSpans; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.LowerCaseTokenizer; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; + +import java.io.Reader; +import java.io.IOException; + + +/** + * + * + **/ +public class BoostingMaxTermQueryTest extends LuceneTestCase { + private IndexSearcher searcher; + private BoostingSimilarity similarity = new BoostingSimilarity(); + private byte[] payloadField = new byte[]{1}; + private byte[] payloadMultiField1 = new byte[]{2}; + private byte[] payloadMultiField2 = new byte[]{4}; + + public BoostingMaxTermQueryTest(String s) { + super(s); + } + + private class PayloadAnalyzer extends Analyzer { + + + public TokenStream tokenStream(String fieldName, Reader reader) { + TokenStream result = new LowerCaseTokenizer(reader); + result = new PayloadFilter(result, fieldName); + return result; + } + } + + private class PayloadFilter extends TokenFilter { + String fieldName; + int numSeen = 0; + + PayloadAttribute payloadAtt; + + public PayloadFilter(TokenStream input, String fieldName) { + super(input); + this.fieldName = fieldName; + payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); + } + + public boolean incrementToken() throws IOException { + boolean hasNext = input.incrementToken(); + if (hasNext) { + if (fieldName.equals("field")) { + payloadAtt.setPayload(new Payload(payloadField)); + } else if (fieldName.equals("multiField")) { + if (numSeen % 2 == 0) { + payloadAtt.setPayload(new Payload(payloadMultiField1)); + } else { + payloadAtt.setPayload(new Payload(payloadMultiField2)); + } + numSeen++; + } + return true; + } else { + return false; + } + } + } + + protected void setUp() throws Exception { + super.setUp(); + RAMDirectory directory = new RAMDirectory(); + PayloadAnalyzer analyzer = new PayloadAnalyzer(); + IndexWriter writer + = new IndexWriter(directory, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); + writer.setSimilarity(similarity); + //writer.infoStream = System.out; + for (int i = 0; i < 1000; i++) { + Document doc = new Document(); + Field noPayloadField = new Field(PayloadHelper.NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED); + //noPayloadField.setBoost(0); + doc.add(noPayloadField); + doc.add(new Field("field", English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED)); + doc.add(new Field("multiField", English.intToEnglish(i) + " " + English.intToEnglish(i), Field.Store.YES, Field.Index.ANALYZED)); + writer.addDocument(doc); + } + writer.optimize(); + writer.close(); + + searcher = new IndexSearcher(directory, true); + searcher.setSimilarity(similarity); + } + + public void test() throws IOException { + BoostingMaxTermQuery query = new BoostingMaxTermQuery(new Term("field", "seventy")); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); + + //they should all have the exact same score, because they all contain seventy once, and we set + //all the other similarity factors to be 1 + + assertTrue(hits.getMaxScore() + " does not equal: " + 1, hits.getMaxScore() == 1); + for (int i = 0; i < hits.scoreDocs.length; i++) { + ScoreDoc doc = hits.scoreDocs[i]; + assertTrue(doc.score + " does not equal: " + 1, doc.score == 1); + } + CheckHits.checkExplanations(query, PayloadHelper.FIELD, searcher, true); + Spans spans = query.getSpans(searcher.getIndexReader()); + assertTrue("spans is null and it shouldn't be", spans != null); + assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); + /*float score = hits.score(0); + for (int i =1; i < hits.length(); i++) + { + assertTrue("scores are not equal and they should be", score == hits.score(i)); + }*/ + + } + + public void testMultipleMatchesPerDoc() throws Exception { + BoostingMaxTermQuery query = new BoostingMaxTermQuery(new Term(PayloadHelper.MULTI_FIELD, "seventy")); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 100, hits.totalHits == 100); + + //they should all have the exact same score, because they all contain seventy once, and we set + //all the other similarity factors to be 1 + + //System.out.println("Hash: " + seventyHash + " Twice Hash: " + 2*seventyHash); + assertTrue(hits.getMaxScore() + " does not equal: " + 4.0, hits.getMaxScore() == 4.0); + //there should be exactly 10 items that score a 4, all the rest should score a 2 + //The 10 items are: 70 + i*100 where i in [0-9] + int numTens = 0; + for (int i = 0; i < hits.scoreDocs.length; i++) { + ScoreDoc doc = hits.scoreDocs[i]; + if (doc.doc % 10 == 0) { + numTens++; + assertTrue(doc.score + " does not equal: " + 4.0, doc.score == 4.0); + } else { + assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); + } + } + assertTrue(numTens + " does not equal: " + 10, numTens == 10); + CheckHits.checkExplanations(query, "field", searcher, true); + Spans spans = query.getSpans(searcher.getIndexReader()); + assertTrue("spans is null and it shouldn't be", spans != null); + assertTrue("spans is not an instanceof " + TermSpans.class, spans instanceof TermSpans); + //should be two matches per document + int count = 0; + //100 hits times 2 matches per hit, we should have 200 in count + while (spans.next()) { + count++; + } + assertTrue(count + " does not equal: " + 200, count == 200); + } + + public void testNoMatch() throws Exception { + BoostingMaxTermQuery query = new BoostingMaxTermQuery(new Term(PayloadHelper.FIELD, "junk")); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 0, hits.totalHits == 0); + + } + + public void testNoPayload() throws Exception { + BoostingMaxTermQuery q1 = new BoostingMaxTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "zero")); + BoostingMaxTermQuery q2 = new BoostingMaxTermQuery(new Term(PayloadHelper.NO_PAYLOAD_FIELD, "foo")); + BooleanClause c1 = new BooleanClause(q1, BooleanClause.Occur.MUST); + BooleanClause c2 = new BooleanClause(q2, BooleanClause.Occur.MUST_NOT); + BooleanQuery query = new BooleanQuery(); + query.add(c1); + query.add(c2); + TopDocs hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("hits Size: " + hits.totalHits + " is not: " + 1, hits.totalHits == 1); + int[] results = new int[1]; + results[0] = 0;//hits.scoreDocs[0].doc; + CheckHits.checkHitCollector(query, PayloadHelper.NO_PAYLOAD_FIELD, searcher, results); + } + + // must be static for weight serialization tests + static class BoostingSimilarity extends DefaultSimilarity { + + // TODO: Remove warning after API has been finalized + public float scorePayload(String fieldName, byte[] payload, int offset, int length) { + //we know it is size 4 here, so ignore the offset/length + return payload[0]; + } + + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //Make everything else 1 so we see the effect of the payload + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + public float lengthNorm(String fieldName, int numTerms) { + return 1; + } + + public float queryNorm(float sumOfSquaredWeights) { + return 1; + } + + public float sloppyFreq(int distance) { + return 1; + } + + public float coord(int overlap, int maxOverlap) { + return 1; + } + + public float idf(int docFreq, int numDocs) { + return 1; + } + + public float tf(float freq) { + return freq == 0 ? 0 : 1; + } + } + +} Property changes on: src/test/org/apache/lucene/search/payloads/BoostingMaxTermQueryTest.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (revision 801868) +++ src/java/org/apache/lucene/search/payloads/BoostingTermQuery.java (working copy) @@ -64,9 +64,9 @@ //TODO: is this the best way to allocate this? byte[] payload = new byte[256]; - private TermPositions positions; + protected TermPositions positions; protected float payloadScore; - private int payloadsSeen; + protected int payloadsSeen; public BoostingSpanScorer(TermSpans spans, QueryWeight weight, Similarity similarity, byte[] norms) throws IOException { @@ -110,10 +110,24 @@ public float score() throws IOException { - return super.score() * (payloadsSeen > 0 ? (payloadScore / payloadsSeen) : 1); + return getSpanScore() * getPayloadScore(); } + /** + * Returns the SpanScorer score only. + *

+ * Should not be overriden without good cause! + * + * @return the score for just the Span part w/o the payload + * @throws IOException + * + * @see #score() + */ + protected float getSpanScore() throws IOException{ + return super.score(); + } + public Explanation explain(final int doc) throws IOException { ComplexExplanation result = new ComplexExplanation(); Explanation nonPayloadExpl = super.explain(doc); @@ -128,15 +142,19 @@ } */ - float avgPayloadScore = (payloadsSeen > 0 ? (payloadScore / payloadsSeen) : 1); - payloadBoost.setValue(avgPayloadScore); + float payloadScore = getPayloadScore(); + payloadBoost.setValue(payloadScore); //GSI: I suppose we could toString the payload, but I don't think that would be a good idea payloadBoost.setDescription("scorePayload(...)"); - result.setValue(nonPayloadExpl.getValue() * avgPayloadScore); + result.setValue(nonPayloadExpl.getValue() * payloadScore); result.setDescription("btq, product of:"); result.setMatch(nonPayloadExpl.getValue()==0 ? Boolean.FALSE : Boolean.TRUE); // LUCENE-1303 return result; } + + protected float getPayloadScore() { + return (payloadsSeen > 0 ? (payloadScore / payloadsSeen) : 1); + } } } Index: src/java/org/apache/lucene/search/payloads/BoostingMaxTermQuery.java =================================================================== --- src/java/org/apache/lucene/search/payloads/BoostingMaxTermQuery.java (revision 0) +++ src/java/org/apache/lucene/search/payloads/BoostingMaxTermQuery.java (revision 0) @@ -0,0 +1,82 @@ +package org.apache.lucene.search.payloads; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Searcher; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.QueryWeight; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.spans.TermSpans; + +import java.io.IOException; + + +/** + * The score returned is based on the maximum payload score seen for the Term on the document, as opposed + * to the average as implemented by {@link org.apache.lucene.search.payloads.BoostingTermQuery}. + * + **/ +public class BoostingMaxTermQuery extends BoostingTermQuery{ + + + public BoostingMaxTermQuery(Term term) { + super(term); + } + + public QueryWeight createQueryWeight(Searcher searcher) throws IOException { + return new BoostingMaxTermWeight(this, searcher); + } + + protected class BoostingMaxTermWeight extends BoostingTermWeight{ + + public BoostingMaxTermWeight(BoostingTermQuery query, Searcher searcher) throws IOException { + super(query, searcher); + } + + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { + return new BoostingMaxSpanScorer((TermSpans) query.getSpans(reader), this, + similarity, reader.norms(query.getField())); + } + + protected class BoostingMaxSpanScorer extends BoostingSpanScorer{ + + public BoostingMaxSpanScorer(TermSpans spans, QueryWeight weight, Similarity similarity, + byte[] norms) throws IOException { + super(spans, weight, similarity, norms); + } + + protected void processPayload(Similarity similarity) throws IOException { + if (positions.isPayloadAvailable()) { + payload = positions.getPayload(payload, 0); + payloadScore = Math.max(payloadScore, similarity.scorePayload(term.field(), payload, 0, positions.getPayloadLength())); + payloadsSeen++; + + } else { + //zero out the payload? + } + } + + protected float getPayloadScore() { + return (payloadsSeen > 0 ? (payloadScore) : 1); + } + + + } + } +} Property changes on: src/java/org/apache/lucene/search/payloads/BoostingMaxTermQuery.java ___________________________________________________________________ Added: svn:eol-style + native