Index: src/java/org/apache/lucene/search/payloads/PayloadFunction.java =================================================================== --- src/java/org/apache/lucene/search/payloads/PayloadFunction.java (revision 946149) +++ src/java/org/apache/lucene/search/payloads/PayloadFunction.java (working copy) @@ -17,16 +17,17 @@ */ import java.io.Serializable; +import org.apache.lucene.search.Explanation; /** - * An abstract class that defines a way for Payload*Query instances to transform - * the cumulative effects of payload scores for a document. - * + * An abstract class that defines a way for Payload*Query instances + * to transform the cumulative effects of payload scores for a document. + * * @see org.apache.lucene.search.payloads.PayloadTermQuery for more information - * - * @lucene.experimental This class and its derivations are experimental and subject to - * change - * + * + *

+ * This class and its derivations are experimental and subject to change + * **/ public abstract class PayloadFunction implements Serializable { @@ -55,10 +56,15 @@ */ public abstract float docScore(int docId, String field, int numPayloadsSeen, float payloadScore); - @Override + public Explanation explain(int docId, int numPayloadsSeen, float payloadScore){ + Explanation result = new Explanation(); + result.setDescription("Unimpl Payload Function Explain"); + result.setValue(1); + return result; + }; + public abstract int hashCode(); - @Override public abstract boolean equals(Object o); } Index: src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java =================================================================== --- src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (revision 946149) +++ src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (working copy) @@ -1,5 +1,7 @@ package org.apache.lucene.search.payloads; +import org.apache.lucene.search.Explanation; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -25,21 +27,21 @@ * **/ public class MaxPayloadFunction extends PayloadFunction { - @Override + public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) { - if (numPayloadsSeen == 0) { - return currentPayloadScore; - } else { - return Math.max(currentPayloadScore, currentScore); - } + return Math.max(currentPayloadScore, currentScore); } - @Override public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) { - return numPayloadsSeen > 0 ? payloadScore : 1; + return numPayloadsSeen > 0 ? payloadScore : 1; } - - @Override + public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) { + Explanation expl = new Explanation(); + float maxPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1); + expl.setValue(maxPayloadScore); + expl.setDescription("MaxPayloadFunction(...)"); + return expl; + } public int hashCode() { final int prime = 31; int result = 1; @@ -47,7 +49,6 @@ return result; } - @Override public boolean equals(Object obj) { if (this == obj) return true; Index: src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java =================================================================== --- src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (revision 946149) +++ src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (working copy) @@ -1,5 +1,7 @@ package org.apache.lucene.search.payloads; +import org.apache.lucene.search.Explanation; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -36,7 +38,13 @@ public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) { return numPayloadsSeen > 0 ? payloadScore : 1; } - + public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) { + Explanation expl = new Explanation(); + float minPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1); + expl.setValue(minPayloadScore); + expl.setDescription("MinPayloadFunction(...)"); + return expl; + } @Override public int hashCode() { final int prime = 31; Index: src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java =================================================================== --- src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (revision 946149) +++ src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (working copy) @@ -43,7 +43,7 @@ * {@link org.apache.lucene.search.spans.TermSpans} occurs. *

* In order to take advantage of this, you must override - * {@link org.apache.lucene.search.Similarity#scorePayload} + * {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)} * which returns 1 by default. *

* Payload scores are aggregated using a pluggable {@link PayloadFunction}. @@ -48,7 +48,8 @@ *

* Payload scores are aggregated using a pluggable {@link PayloadFunction}. * - * @see org.apache.lucene.search.Similarity#scorePayload + * @see org.apache.lucene.search.Similarity#scorePayload(String, byte[], int, + * int) */ public class PayloadNearQuery extends SpanNearQuery { protected String fieldName; @@ -65,7 +66,6 @@ this.function = function; } - @Override public Weight createWeight(Searcher searcher) throws IOException { return new PayloadNearSpanWeight(this, searcher); } @@ -70,7 +70,6 @@ return new PayloadNearSpanWeight(this, searcher); } - @Override public Object clone() { int sz = clauses.size(); SpanQuery[] newClauses = new SpanQuery[sz]; @@ -76,10 +75,11 @@ SpanQuery[] newClauses = new SpanQuery[sz]; for (int i = 0; i < sz; i++) { - newClauses[i] = (SpanQuery) clauses.get(i).clone(); + SpanQuery clause = (SpanQuery) clauses.get(i); + newClauses[i] = (SpanQuery) clause.clone(); } PayloadNearQuery boostingNearQuery = new PayloadNearQuery(newClauses, slop, - inOrder); + inOrder, function); boostingNearQuery.setBoost(getBoost()); return boostingNearQuery; } @@ -84,13 +84,12 @@ return boostingNearQuery; } - @Override public String toString(String field) { - StringBuilder buffer = new StringBuilder(); + StringBuffer buffer = new StringBuffer(); buffer.append("payloadNear(["); - Iterator i = clauses.iterator(); + Iterator i = clauses.iterator(); while (i.hasNext()) { - SpanQuery clause = i.next(); + SpanQuery clause = (SpanQuery) i.next(); buffer.append(clause.toString(field)); if (i.hasNext()) { buffer.append(", "); @@ -105,7 +104,7 @@ return buffer.toString(); } - @Override + // @Override public int hashCode() { final int prime = 31; int result = super.hashCode(); @@ -114,7 +113,7 @@ return result; } - @Override + // @Override public boolean equals(Object obj) { if (this == obj) return true; @@ -142,7 +141,11 @@ super(query, searcher); } - @Override + public Scorer scorer(IndexReader reader) throws IOException { + return new PayloadNearSpanScorer(query.getSpans(reader), this, + similarity, reader.norms(query.getField())); + } + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException { return new PayloadNearSpanScorer(query.getSpans(reader), this, @@ -152,7 +155,6 @@ public class PayloadNearSpanScorer extends SpanScorer { Spans spans; - protected float payloadScore; private int payloadsSeen; Similarity similarity = getSimilarity(); @@ -192,8 +194,9 @@ * * @see Spans */ - protected void processPayloads(Collection payLoads, int start, int end) { - for (final byte[] thePayload : payLoads) { + protected void processPayloads(Collection payLoads, int start, int end) { + for (Iterator iterator = payLoads.iterator(); iterator.hasNext();) { + byte[] thePayload = (byte[]) iterator.next(); payloadScore = function.currentScore(doc, fieldName, start, end, payloadsSeen, payloadScore, similarity.scorePayload(doc, fieldName, spans.start(), spans.end(), thePayload, 0, thePayload.length)); @@ -202,20 +205,25 @@ } // - @Override protected boolean setFreqCurrentDoc() throws IOException { - if (!more) { - return false; - } - Spans[] spansArr = new Spans[1]; - spansArr[0] = spans; - payloadScore = 0; - payloadsSeen = 0; - getPayloads(spansArr); - return super.setFreqCurrentDoc(); + if (!more) { + return false; + } + doc = spans.doc(); + freq = 0.0f; + payloadScore = 0; + payloadsSeen = 0; + do { + int matchLength = spans.end() - spans.start(); + freq += getSimilarity().sloppyFreq(matchLength); + Spans[] spansArr = new Spans[1]; + spansArr[0] = spans; + getPayloads(spansArr); + more = spans.next(); + } while (more && (doc == spans.doc())); + return true; } - @Override public float score() throws IOException { return super.score() @@ -222,19 +230,16 @@ * function.docScore(doc, fieldName, payloadsSeen, payloadScore); } - @Override - protected Explanation explain(int doc) throws IOException { + public Explanation explain(int doc) throws IOException { Explanation result = new Explanation(); + // Add detail about tf/idf... Explanation nonPayloadExpl = super.explain(doc); result.addDetail(nonPayloadExpl); - Explanation payloadBoost = new Explanation(); - result.addDetail(payloadBoost); - float avgPayloadScore = (payloadsSeen > 0 ? (payloadScore / payloadsSeen) - : 1); - payloadBoost.setValue(avgPayloadScore); - payloadBoost.setDescription("scorePayload(...)"); - result.setValue(nonPayloadExpl.getValue() * avgPayloadScore); - result.setDescription("bnq, product of:"); + // Add detail about payload + Explanation payloadExpl = function.explain(doc, payloadsSeen, payloadScore); + result.addDetail(payloadExpl); + result.setValue(nonPayloadExpl.getValue() * payloadExpl.getValue()); + result.setDescription("PayloadNearQuery, product of:"); return result; } } Index: src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java =================================================================== --- src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (revision 946149) +++ src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (working copy) @@ -1,5 +1,9 @@ package org.apache.lucene.search.payloads; +import java.io.IOException; + +import org.apache.lucene.search.Explanation; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -26,7 +30,6 @@ **/ public class AveragePayloadFunction extends PayloadFunction{ - @Override public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) { return currentPayloadScore + currentScore; } @@ -31,12 +34,18 @@ return currentPayloadScore + currentScore; } - @Override public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) { return numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1; } + + public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) { + Explanation expl = new Explanation(); + float avgPayloadScore = (numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1); + expl.setValue(avgPayloadScore); + expl.setDescription("AveragePayloadFunction(...)"); + return expl; + } - @Override public int hashCode() { final int prime = 31; int result = 1; @@ -44,7 +53,6 @@ return result; } - @Override public boolean equals(Object obj) { if (this == obj) return true; Index: src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java =================================================================== --- src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 946161) +++ src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy) @@ -31,7 +31,9 @@ import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Searcher; @@ -65,6 +67,7 @@ } private class PayloadFilter extends TokenFilter { + String fieldName; int numSeen = 0; protected PayloadAttribute payAtt; @@ -70,7 +73,8 @@ public PayloadFilter(TokenStream input, String fieldName) { super(input); - payAtt = addAttribute(PayloadAttribute.class); + this.fieldName = fieldName; + payAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); } @Override @@ -89,7 +93,7 @@ } } - private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder) { + private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder, PayloadFunction function ) { String[] words = phrase.split("[\\s]+"); SpanQuery clauses[] = new SpanQuery[words.length]; for (int i=0;i -1); + assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 3, explain.getValue() == 3f); + } + } + public void testMaxFunction() throws IOException { + PayloadNearQuery query; + TopDocs hits; + + query = newPhraseQuery("field", "twenty two", true, new MaxPayloadFunction()); + QueryUtils.check(query); + // all 10 hits should have score = 4 (max payload value) + hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("should be 10 hits", hits.totalHits == 10); + for (int j = 0; j < hits.scoreDocs.length; j++) { + ScoreDoc doc = hits.scoreDocs[j]; + assertTrue(doc.score + " does not equal: " + 4, doc.score == 4); + Explanation explain = searcher.explain(query, hits.scoreDocs[j].doc); + String exp = explain.toString(); + assertTrue(exp, exp.indexOf("MaxPayloadFunction") > -1); + assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 4, explain.getValue() == 4f); + } + } + public void testMinFunction() throws IOException { + PayloadNearQuery query; + TopDocs hits; + query = newPhraseQuery("field", "twenty two", true, new MinPayloadFunction()); + QueryUtils.check(query); + // all 10 hits should have score = 2 (min payload value) + hits = searcher.search(query, null, 100); + assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("should be 10 hits", hits.totalHits == 10); + for (int j = 0; j < hits.scoreDocs.length; j++) { + ScoreDoc doc = hits.scoreDocs[j]; + assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); + Explanation explain = searcher.explain(query, hits.scoreDocs[j].doc); + String exp = explain.toString(); + assertTrue(exp, exp.indexOf("MinPayloadFunction") > -1); + assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 2, explain.getValue() == 2f); + } + } + private SpanQuery[] getClauses() { + SpanNearQuery q1, q2; + q1 = spanNearQuery("field2", "twenty two"); + q2 = spanNearQuery("field2", "twenty three"); + SpanQuery[] clauses = new SpanQuery[2]; + clauses[0] = q1; + clauses[1] = q2; + return clauses; + } private SpanNearQuery spanNearQuery(String fieldName, String words) { String[] wordList = words.split("[\\s]+"); SpanQuery clauses[] = new SpanQuery[wordList.length]; @@ -185,12 +258,12 @@ public void testLongerSpan() throws IOException { PayloadNearQuery query; TopDocs hits; - query = newPhraseQuery("field", "nine hundred ninety nine", true); + query = newPhraseQuery("field", "nine hundred ninety nine", true, new AveragePayloadFunction()); hits = searcher.search(query, null, 100); - assertTrue("hits is null and it shouldn't be", hits != null); ScoreDoc doc = hits.scoreDocs[0]; // System.out.println("Doc: " + doc.toString()); // System.out.println("Explain: " + searcher.explain(query, doc.doc)); + assertTrue("hits is null and it shouldn't be", hits != null); assertTrue("there should only be one hit", hits.totalHits == 1); // should have score = 3 because adjacent terms have payloads of 2,4 assertTrue(doc.score + " does not equal: " + 3, doc.score == 3); @@ -202,10 +275,10 @@ // combine ordered and unordered spans with some nesting to make sure all payloads are counted - SpanQuery q1 = newPhraseQuery("field", "nine hundred", true); - SpanQuery q2 = newPhraseQuery("field", "ninety nine", true); - SpanQuery q3 = newPhraseQuery("field", "nine ninety", false); - SpanQuery q4 = newPhraseQuery("field", "hundred nine", false); + SpanQuery q1 = newPhraseQuery("field", "nine hundred", true, new AveragePayloadFunction()); + SpanQuery q2 = newPhraseQuery("field", "ninety nine", true, new AveragePayloadFunction()); + SpanQuery q3 = newPhraseQuery("field", "nine ninety", false, new AveragePayloadFunction()); + SpanQuery q4 = newPhraseQuery("field", "hundred nine", false, new AveragePayloadFunction()); SpanQuery[]clauses = new SpanQuery[] {new PayloadNearQuery(new SpanQuery[] {q1,q2}, 0, true), new PayloadNearQuery(new SpanQuery[] {q3,q4}, 0, false)}; query = new PayloadNearQuery(clauses, 0, false); hits = searcher.search(query, null, 100); @@ -226,7 +299,6 @@ //we know it is size 4 here, so ignore the offset/length return payload[0]; } - //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -248,7 +320,6 @@ @Override public float tf(float freq) { return 1.0f; } - // idf used for phrase queries @Override public IDFExplanation idfExplain(Collection terms, Searcher searcher) throws IOException { return new IDFExplanation() { Index: PNQ-patch1.txt =================================================================== --- PNQ-patch1.txt (revision 0) +++ PNQ-patch1.txt (revision 0) @@ -0,0 +1,548 @@ +Index: src/java/org/apache/lucene/search/payloads/PayloadFunction.java +=================================================================== +--- src/java/org/apache/lucene/search/payloads/PayloadFunction.java (revision 946149) ++++ src/java/org/apache/lucene/search/payloads/PayloadFunction.java (working copy) +@@ -17,16 +17,17 @@ + */ + + import java.io.Serializable; ++import org.apache.lucene.search.Explanation; + + /** +- * An abstract class that defines a way for Payload*Query instances to transform +- * the cumulative effects of payload scores for a document. +- * ++ * An abstract class that defines a way for Payload*Query instances ++ * to transform the cumulative effects of payload scores for a document. ++ * + * @see org.apache.lucene.search.payloads.PayloadTermQuery for more information +- * +- * @lucene.experimental This class and its derivations are experimental and subject to +- * change +- * ++ * ++ *

++ * This class and its derivations are experimental and subject to change ++ * + **/ + public abstract class PayloadFunction implements Serializable { + +@@ -55,10 +56,15 @@ + */ + public abstract float docScore(int docId, String field, int numPayloadsSeen, float payloadScore); + +- @Override ++ public Explanation explain(int docId, int numPayloadsSeen, float payloadScore){ ++ Explanation result = new Explanation(); ++ result.setDescription("Unimpl Payload Function Explain"); ++ result.setValue(1); ++ return result; ++ }; ++ + public abstract int hashCode(); + +- @Override + public abstract boolean equals(Object o); + + } +Index: src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java +=================================================================== +--- src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (revision 946149) ++++ src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (working copy) +@@ -1,5 +1,7 @@ + package org.apache.lucene.search.payloads; + ++import org.apache.lucene.search.Explanation; ++ + /** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with +@@ -25,21 +27,21 @@ + * + **/ + public class MaxPayloadFunction extends PayloadFunction { +- @Override ++ + public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) { +- if (numPayloadsSeen == 0) { +- return currentPayloadScore; +- } else { +- return Math.max(currentPayloadScore, currentScore); +- } ++ return Math.max(currentPayloadScore, currentScore); + } + +- @Override + public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) { +- return numPayloadsSeen > 0 ? payloadScore : 1; ++ return numPayloadsSeen > 0 ? payloadScore : 1; + } +- +- @Override ++ public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) { ++ Explanation payloadBoost = new Explanation(); ++ float maxPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1); ++ payloadBoost.setValue(maxPayloadScore); ++ payloadBoost.setDescription("MaxPayloadFunction(...)"); ++ return payloadBoost; ++ } + public int hashCode() { + final int prime = 31; + int result = 1; +@@ -47,7 +49,6 @@ + return result; + } + +- @Override + public boolean equals(Object obj) { + if (this == obj) + return true; +Index: src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java +=================================================================== +--- src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (revision 946149) ++++ src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (working copy) +@@ -1,5 +1,7 @@ + package org.apache.lucene.search.payloads; + ++import org.apache.lucene.search.Explanation; ++ + /** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with +Index: src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java +=================================================================== +--- src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (revision 946149) ++++ src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (working copy) +@@ -43,7 +43,7 @@ + * {@link org.apache.lucene.search.spans.TermSpans} occurs. + *

+ * In order to take advantage of this, you must override +- * {@link org.apache.lucene.search.Similarity#scorePayload} ++ * {@link org.apache.lucene.search.Similarity#scorePayload(String, byte[],int,int)} + * which returns 1 by default. + *

+ * Payload scores are aggregated using a pluggable {@link PayloadFunction}. +@@ -48,7 +48,8 @@ + *

+ * Payload scores are aggregated using a pluggable {@link PayloadFunction}. + * +- * @see org.apache.lucene.search.Similarity#scorePayload ++ * @see org.apache.lucene.search.Similarity#scorePayload(String, byte[], int, ++ * int) + */ + public class PayloadNearQuery extends SpanNearQuery { + protected String fieldName; +@@ -65,7 +66,6 @@ + this.function = function; + } + +- @Override + public Weight createWeight(Searcher searcher) throws IOException { + return new PayloadNearSpanWeight(this, searcher); + } +@@ -70,7 +70,6 @@ + return new PayloadNearSpanWeight(this, searcher); + } + +- @Override + public Object clone() { + int sz = clauses.size(); + SpanQuery[] newClauses = new SpanQuery[sz]; +@@ -76,10 +75,11 @@ + SpanQuery[] newClauses = new SpanQuery[sz]; + + for (int i = 0; i < sz; i++) { +- newClauses[i] = (SpanQuery) clauses.get(i).clone(); ++ SpanQuery clause = (SpanQuery) clauses.get(i); ++ newClauses[i] = (SpanQuery) clause.clone(); + } + PayloadNearQuery boostingNearQuery = new PayloadNearQuery(newClauses, slop, +- inOrder); ++ inOrder, function); + boostingNearQuery.setBoost(getBoost()); + return boostingNearQuery; + } +@@ -84,13 +84,12 @@ + return boostingNearQuery; + } + +- @Override + public String toString(String field) { +- StringBuilder buffer = new StringBuilder(); ++ StringBuffer buffer = new StringBuffer(); + buffer.append("payloadNear(["); +- Iterator i = clauses.iterator(); ++ Iterator i = clauses.iterator(); + while (i.hasNext()) { +- SpanQuery clause = i.next(); ++ SpanQuery clause = (SpanQuery) i.next(); + buffer.append(clause.toString(field)); + if (i.hasNext()) { + buffer.append(", "); +@@ -105,7 +104,7 @@ + return buffer.toString(); + } + +- @Override ++ // @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); +@@ -114,7 +113,7 @@ + return result; + } + +- @Override ++ // @Override + public boolean equals(Object obj) { + if (this == obj) + return true; +@@ -142,7 +141,11 @@ + super(query, searcher); + } + +- @Override ++ public Scorer scorer(IndexReader reader) throws IOException { ++ return new PayloadNearSpanScorer(query.getSpans(reader), this, ++ similarity, reader.norms(query.getField())); ++ } ++ + public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, + boolean topScorer) throws IOException { + return new PayloadNearSpanScorer(query.getSpans(reader), this, +@@ -152,7 +155,6 @@ + + public class PayloadNearSpanScorer extends SpanScorer { + Spans spans; +- + protected float payloadScore; + private int payloadsSeen; + Similarity similarity = getSimilarity(); +@@ -192,8 +194,9 @@ + * + * @see Spans + */ +- protected void processPayloads(Collection payLoads, int start, int end) { +- for (final byte[] thePayload : payLoads) { ++ protected void processPayloads(Collection payLoads, int start, int end) { ++ for (Iterator iterator = payLoads.iterator(); iterator.hasNext();) { ++ byte[] thePayload = (byte[]) iterator.next(); + payloadScore = function.currentScore(doc, fieldName, start, end, + payloadsSeen, payloadScore, similarity.scorePayload(doc, fieldName, + spans.start(), spans.end(), thePayload, 0, thePayload.length)); +@@ -202,20 +205,25 @@ + } + + // +- @Override + protected boolean setFreqCurrentDoc() throws IOException { +- if (!more) { +- return false; +- } +- Spans[] spansArr = new Spans[1]; +- spansArr[0] = spans; +- payloadScore = 0; +- payloadsSeen = 0; +- getPayloads(spansArr); +- return super.setFreqCurrentDoc(); ++ if (!more) { ++ return false; ++ } ++ doc = spans.doc(); ++ freq = 0.0f; ++ payloadScore = 0; ++ payloadsSeen = 0; ++ do { ++ int matchLength = spans.end() - spans.start(); ++ freq += getSimilarity().sloppyFreq(matchLength); ++ Spans[] spansArr = new Spans[1]; ++ spansArr[0] = spans; ++ getPayloads(spansArr); ++ more = spans.next(); ++ } while (more && (doc == spans.doc())); ++ return true; + } + +- @Override + public float score() throws IOException { + + return super.score() +@@ -222,19 +230,16 @@ + * function.docScore(doc, fieldName, payloadsSeen, payloadScore); + } + +- @Override +- protected Explanation explain(int doc) throws IOException { ++ public Explanation explain(int doc) throws IOException { + Explanation result = new Explanation(); ++ // Add detail about tf/idf... + Explanation nonPayloadExpl = super.explain(doc); + result.addDetail(nonPayloadExpl); +- Explanation payloadBoost = new Explanation(); +- result.addDetail(payloadBoost); +- float avgPayloadScore = (payloadsSeen > 0 ? (payloadScore / payloadsSeen) +- : 1); +- payloadBoost.setValue(avgPayloadScore); +- payloadBoost.setDescription("scorePayload(...)"); +- result.setValue(nonPayloadExpl.getValue() * avgPayloadScore); +- result.setDescription("bnq, product of:"); ++ // Add detail about payload ++ Explanation payloadExpl = function.explain(doc, payloadsSeen, payloadScore); ++ result.addDetail(payloadExpl); ++ result.setValue(nonPayloadExpl.getValue() * payloadExpl.getValue()); ++ result.setDescription("PayloadNearQuery, product of:"); + return result; + } + } +Index: src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java +=================================================================== +--- src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (revision 946149) ++++ src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (working copy) +@@ -1,5 +1,9 @@ + package org.apache.lucene.search.payloads; + ++import java.io.IOException; ++ ++import org.apache.lucene.search.Explanation; ++ + /** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with +@@ -26,7 +30,6 @@ + **/ + public class AveragePayloadFunction extends PayloadFunction{ + +- @Override + public float currentScore(int docId, String field, int start, int end, int numPayloadsSeen, float currentScore, float currentPayloadScore) { + return currentPayloadScore + currentScore; + } +@@ -31,12 +34,18 @@ + return currentPayloadScore + currentScore; + } + +- @Override + public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) { + return numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1; + } ++ ++ public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) { ++ Explanation payloadBoost = new Explanation(); ++ float avgPayloadScore = (numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1); ++ payloadBoost.setValue(avgPayloadScore); ++ payloadBoost.setDescription("AveragePayloadFunction(...)"); ++ return payloadBoost; ++ } + +- @Override + public int hashCode() { + final int prime = 31; + int result = 1; +@@ -44,7 +53,6 @@ + return result; + } + +- @Override + public boolean equals(Object obj) { + if (this == obj) + return true; +Index: src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java +=================================================================== +--- src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 946161) ++++ src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy) +@@ -31,7 +31,9 @@ + import org.apache.lucene.index.Payload; + import org.apache.lucene.index.Term; + import org.apache.lucene.search.DefaultSimilarity; ++import org.apache.lucene.search.Explanation; + import org.apache.lucene.search.IndexSearcher; ++import org.apache.lucene.search.Query; + import org.apache.lucene.search.QueryUtils; + import org.apache.lucene.search.ScoreDoc; + import org.apache.lucene.search.Searcher; +@@ -65,6 +67,7 @@ + } + + private class PayloadFilter extends TokenFilter { ++ String fieldName; + int numSeen = 0; + protected PayloadAttribute payAtt; + +@@ -70,7 +73,8 @@ + + public PayloadFilter(TokenStream input, String fieldName) { + super(input); +- payAtt = addAttribute(PayloadAttribute.class); ++ this.fieldName = fieldName; ++ payAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); + } + + @Override +@@ -89,7 +93,7 @@ + } + } + +- private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder) { ++ private PayloadNearQuery newPhraseQuery (String fieldName, String phrase, boolean inOrder, PayloadFunction function ) { + String[] words = phrase.split("[\\s]+"); + SpanQuery clauses[] = new SpanQuery[words.length]; + for (int i=0;i -1); ++ assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 3, explain.getValue() == 3f); ++ } ++ } ++ public void testMaxFunction() throws IOException { ++ PayloadNearQuery query; ++ TopDocs hits; ++ ++ query = newPhraseQuery("field", "twenty two", true, new MaxPayloadFunction()); ++ QueryUtils.check(query); ++ // all 10 hits should have score = 4 (max payload value) ++ hits = searcher.search(query, null, 100); ++ assertTrue("hits is null and it shouldn't be", hits != null); ++ assertTrue("should be 10 hits", hits.totalHits == 10); ++ for (int j = 0; j < hits.scoreDocs.length; j++) { ++ ScoreDoc doc = hits.scoreDocs[j]; ++ assertTrue(doc.score + " does not equal: " + 4, doc.score == 4); ++ Explanation explain = searcher.explain(query, hits.scoreDocs[j].doc); ++ String exp = explain.toString(); ++ assertTrue(exp, exp.indexOf("MaxPayloadFunction") > -1); ++ assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 4, explain.getValue() == 4f); ++ } ++ } ++ public void testMinFunction() throws IOException { ++ PayloadNearQuery query; ++ TopDocs hits; + ++ query = newPhraseQuery("field", "twenty two", true, new MinPayloadFunction()); ++ QueryUtils.check(query); ++ // all 10 hits should have score = 2 (min payload value) ++ hits = searcher.search(query, null, 100); ++ assertTrue("hits is null and it shouldn't be", hits != null); ++ assertTrue("should be 10 hits", hits.totalHits == 10); ++ for (int j = 0; j < hits.scoreDocs.length; j++) { ++ ScoreDoc doc = hits.scoreDocs[j]; ++ assertTrue(doc.score + " does not equal: " + 2, doc.score == 2); ++ Explanation explain = searcher.explain(query, hits.scoreDocs[j].doc); ++ String exp = explain.toString(); ++ assertTrue(exp, exp.indexOf("MinPayloadFunction") > -1); ++ assertTrue(hits.scoreDocs[j].score + " explain value does not equal: " + 2, explain.getValue() == 2f); ++ } ++ } ++ private SpanQuery[] getClauses() { ++ SpanNearQuery q1, q2; ++ q1 = spanNearQuery("field2", "twenty two"); ++ q2 = spanNearQuery("field2", "twenty three"); ++ SpanQuery[] clauses = new SpanQuery[2]; ++ clauses[0] = q1; ++ clauses[1] = q2; ++ return clauses; ++ } + private SpanNearQuery spanNearQuery(String fieldName, String words) { + String[] wordList = words.split("[\\s]+"); + SpanQuery clauses[] = new SpanQuery[wordList.length]; +@@ -185,12 +258,12 @@ + public void testLongerSpan() throws IOException { + PayloadNearQuery query; + TopDocs hits; +- query = newPhraseQuery("field", "nine hundred ninety nine", true); ++ query = newPhraseQuery("field", "nine hundred ninety nine", true, new AveragePayloadFunction()); + hits = searcher.search(query, null, 100); +- assertTrue("hits is null and it shouldn't be", hits != null); + ScoreDoc doc = hits.scoreDocs[0]; + // System.out.println("Doc: " + doc.toString()); + // System.out.println("Explain: " + searcher.explain(query, doc.doc)); ++ assertTrue("hits is null and it shouldn't be", hits != null); + assertTrue("there should only be one hit", hits.totalHits == 1); + // should have score = 3 because adjacent terms have payloads of 2,4 + assertTrue(doc.score + " does not equal: " + 3, doc.score == 3); +@@ -202,10 +275,10 @@ + + // combine ordered and unordered spans with some nesting to make sure all payloads are counted + +- SpanQuery q1 = newPhraseQuery("field", "nine hundred", true); +- SpanQuery q2 = newPhraseQuery("field", "ninety nine", true); +- SpanQuery q3 = newPhraseQuery("field", "nine ninety", false); +- SpanQuery q4 = newPhraseQuery("field", "hundred nine", false); ++ SpanQuery q1 = newPhraseQuery("field", "nine hundred", true, new AveragePayloadFunction()); ++ SpanQuery q2 = newPhraseQuery("field", "ninety nine", true, new AveragePayloadFunction()); ++ SpanQuery q3 = newPhraseQuery("field", "nine ninety", false, new AveragePayloadFunction()); ++ SpanQuery q4 = newPhraseQuery("field", "hundred nine", false, new AveragePayloadFunction()); + SpanQuery[]clauses = new SpanQuery[] {new PayloadNearQuery(new SpanQuery[] {q1,q2}, 0, true), new PayloadNearQuery(new SpanQuery[] {q3,q4}, 0, false)}; + query = new PayloadNearQuery(clauses, 0, false); + hits = searcher.search(query, null, 100); +@@ -226,7 +299,6 @@ + //we know it is size 4 here, so ignore the offset/length + return payload[0]; + } +- + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + //Make everything else 1 so we see the effect of the payload + //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +@@ -248,7 +320,6 @@ + @Override public float tf(float freq) { + return 1.0f; + } +- + // idf used for phrase queries + @Override public IDFExplanation idfExplain(Collection terms, Searcher searcher) throws IOException { + return new IDFExplanation() {