Index: contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java =================================================================== --- contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (revision 607926) +++ contrib/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java (working copy) @@ -87,8 +87,9 @@ QualityQueryParser qqParser = new SimpleQQParser("title","body"); QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField); - SubmissionReport submitLog = DEBUG ? new SubmissionReport(logger) : null; - QualityStats stats[] = qrun.execute(maxResults, judge, submitLog, logger); + SubmissionReport submitLog = DEBUG ? new SubmissionReport(logger, "TestRun") : null; + qrun.setMaxResults(maxResults); + QualityStats stats[] = qrun.execute(judge, submitLog, logger); // --------- verify by the way judgments were altered for this test: // for some queries, depending on m = qnum % 8 Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java (revision 607926) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/SubmissionReport.java (working copy) @@ -35,13 +35,16 @@ private NumberFormat nf; private PrintWriter logger; + private String name; /** * Constructor for SubmissionReport. * @param logger if null, no submission data is created. + * @param name name of this run. */ - public SubmissionReport (PrintWriter logger) { + public SubmissionReport (PrintWriter logger, String name) { this.logger = logger; + this.name = name; nf = NumberFormat.getInstance(); nf.setMaximumFractionDigits(4); nf.setMinimumFractionDigits(4); @@ -66,14 +69,21 @@ String docName = xt.docName(searcher,sd[i].doc); logger.println( qq.getQueryID() + sep + - '0' + sep + + "Q0" + sep + format(docName,20) + sep + format(""+i,7) + sep + - nf.format(sd[i].score) + nf.format(sd[i].score) + sep + + name ); } } + public void flush() { + if (logger!=null) { + logger.flush(); + } + } + private static String padd = " "; private String format(String s, int minLen) { s = (s==null ? "" : s); Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java (revision 607926) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/QualityBenchmark.java (working copy) @@ -51,6 +51,12 @@ /** index field to extract doc name for each search result; used for judging the results. */ protected String docNameField; + + /** maximal number of queries that this quality benchmark runs. Default: maxint. Useful for debugging. */ + private int maxQueries = Integer.MAX_VALUE; + + /** maximal number of results to collect for each query. Default: 1000. */ + private int maxResults = 1000; /** * Create a QualityBenchmark. @@ -71,7 +77,6 @@ /** * Run the quality benchmark. - * @param maxResults how many results to collect for each quality query. * @param judge the judge that can tell if a certain result doc is relevant for a certain quality query. * If null, no judgements would be made. Usually null for a submission run. * @param submitRep submission report is created if non null. @@ -79,10 +84,11 @@ * @return QualityStats of each quality query that was executed. * @throws Exception if quality benchmark failed to run. */ - public QualityStats [] execute(int maxResults, Judge judge, SubmissionReport submitRep, + public QualityStats [] execute(Judge judge, SubmissionReport submitRep, PrintWriter qualityLog) throws Exception { - QualityStats stats[] = new QualityStats[qualityQueries.length]; - for (int i=0; i + * Reciprocal rank is defined as 1/r where r is the + * rank of the first correct result, or 0 if there are no correct + * results within the top 5 results. + *

+ * This follows the definition in + * + * Question Answering - CNLP at the TREC-10 Question Answering Track. + */ + public double getMRR() { + return mrr; + } + + + /** * Returns the search time in milliseconds for the measured query. */ public long getSearchTime() { Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java (revision 0) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java (revision 0) @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.quality.trec; + +import java.io.BufferedReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; + +import org.apache.lucene.benchmark.quality.QualityQuery; + +/** + * Read topics of TREC 1MQ track. + *

+ * Expects this topic format - + * 

+ *   qnum:qtext
+ *
+ * Comment lines starting with '#' are ignored. + *

+ * All topics will have a single name value pair. + */ +public class Trec1MQReader { + + private String name; + + /** + * Constructor for Trec's 1MQ TopicsReader + * @param name name of name-value pair to set for all queries. + */ + public Trec1MQReader(String name) { + super(); + this.name = name; + } + + /** + * Read quality queries from trec 1MQ format topics file. + * @param reader where queries are read from. + * @return the result quality queries. + * @throws IOException if cannot read the queries. + */ + public QualityQuery[] readQueries(BufferedReader reader) throws IOException { + ArrayList res = new ArrayList(); + String line; + try { + while (null!=(line=reader.readLine())) { + line = line.trim(); + if (line.startsWith("#")) { + continue; + } + // id + int k = line.indexOf(":"); + String id = line.substring(0,k).trim(); + // qtext + String qtext = line.substring(k+1).trim(); + // we got a topic! + HashMap fields = new HashMap(); + fields.put(name,qtext); + //System.out.println("id: "+id+" qtext: "+qtext+" line: "+line); + QualityQuery topic = new QualityQuery(id,fields); + res.add(topic); + } + } finally { + reader.close(); + } + // sort result array (by ID) + QualityQuery qq[] = (QualityQuery[]) res.toArray(new QualityQuery[0]); + Arrays.sort(qq); + return qq; + } + +} Property changes on: contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/Trec1MQReader.java ___________________________________________________________________ Name: svn:executable + * Name: svn:eol-style + native Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java (revision 607926) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java (working copy) @@ -46,8 +46,9 @@ // run the benchmark QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField); + qrun.setMaxResults(maxResults); SubmissionReport submitLog = null; - QualityStats stats[] = qrun.execute(maxResults, judge, submitLog, logger); + QualityStats stats[] = qrun.execute(judge, submitLog, logger); // print an avarage sum of the results QualityStats avg = QualityStats.average(stats);