Index: src/test/org/apache/lucene/search/PostCollectSortCollectorTest.java =================================================================== --- src/test/org/apache/lucene/search/PostCollectSortCollectorTest.java (revision 0) +++ src/test/org/apache/lucene/search/PostCollectSortCollectorTest.java (revision 0) @@ -0,0 +1,75 @@ +package org.apache.lucene.search; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.English; +import org.apache.lucene.util.LuceneTestCaseJ4; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import org.junit.Test; + + +/** + * + * + **/ +public class PostCollectSortCollectorTest extends LuceneTestCaseJ4 { + protected Directory directory; + + @Override + + public void setUp() throws Exception { + super.setUp(); + directory = new RAMDirectory(); + IndexWriter iw = new IndexWriter(directory, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED); + for (int i = 0; i < 1000; i++) { + Document d = new Document(); + d.add(new Field("eng", English.intToEnglish(i), Field.Store.NO, Field.Index.ANALYZED)); + iw.addDocument(d); + } + iw.close(); + } + + @Test + public void testCollect() throws Exception { + Query q = new TermQuery(new Term("eng", "one")); + IndexSearcher searcher = new IndexSearcher(directory, true); + PostCollectSortCollector coll = new PostCollectSortCollector(); + searcher.search(q, coll); + searcher.close(); + int totalHits = coll.getTotalHits(); + assertEquals(totalHits, 109); + TopDocs res = coll.topDocs(); + assertEquals(totalHits, res.totalHits); + //NOTE: Here it is all right to use total hits as the top docs array is the same. + //assert they are in descending order + float lastScore = Float.MAX_VALUE; + for (int i = 0; i < res.totalHits; i++){ + assertTrue(res.scoreDocs[i].score <= lastScore); + lastScore = res.scoreDocs[i].score; + } + + } +} Property changes on: src/test/org/apache/lucene/search/PostCollectSortCollectorTest.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/search/PostCollectSortCollector.java =================================================================== --- src/java/org/apache/lucene/search/PostCollectSortCollector.java (revision 0) +++ src/java/org/apache/lucene/search/PostCollectSortCollector.java (revision 0) @@ -0,0 +1,149 @@ +package org.apache.lucene.search; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.ArrayUtil; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; + + +/** + * Collects all docs and then sorts them after all collection is done. This may be faster when retrieving really large + * result sets at the cost of memory (likely memory that would be used anyway, since a lot of results are being collected). + * + **/ +public class PostCollectSortCollector extends Collector{ + + public static final Comparator DESC_SCORE_DOC_COMPARATOR = new Comparator() { + public int compare(ScoreDoc scoreDoc, ScoreDoc scoreDoc1) { + int result = Float.compare(scoreDoc1.score, scoreDoc.score); + if (result == 0){ + result = scoreDoc1.doc - scoreDoc.doc; + } + return result; + } + }; + public static final Comparator ASC_SCORE_DOC_COMPARATOR = new Comparator() { + public int compare(ScoreDoc scoreDoc, ScoreDoc scoreDoc1) { + int result = Float.compare(scoreDoc.score, scoreDoc1.score); + if (result == 0){ + result = scoreDoc.doc - scoreDoc1.doc; + } + return result; + } + }; + + private ScoreDoc[] docs; + private static final int DEFAULT_INIT_SIZE = 500; + private Scorer scorer; + private int totalHits; + + private Comparator comparator; + + public PostCollectSortCollector() throws IOException { + this(new ScoreDoc[DEFAULT_INIT_SIZE], DESC_SCORE_DOC_COMPARATOR); + } + + public PostCollectSortCollector(int initialSize) throws IOException { + this(new ScoreDoc[initialSize], DESC_SCORE_DOC_COMPARATOR); + } + + public PostCollectSortCollector(int initialSize, Comparator comparator) throws IOException { + this(new ScoreDoc[initialSize], comparator); + } + + /** + * Pass in a pre-allocated array for the score docs. + * @param docs A pre-allocated array + * @param comparator + */ + public PostCollectSortCollector(ScoreDoc[] docs, Comparator comparator) throws IOException { + this.docs = docs; + this.comparator = comparator; + } + + public int getTotalHits() { + return totalHits; + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); + if (score > 0) { + if (totalHits == docs.length) { + docs = ArrayUtil.grow(docs); + } + docs[totalHits++] = new ScoreDoc(doc, score); + } + + } + + @Override + public void postCollect() throws IOException { + Arrays.sort(docs, 0, totalHits, comparator); + } + + @Override + public void setNextReader(IndexReader reader, int docBase) throws IOException { + } + + /** + * Note, the docs array will not be shrunk + * @return + */ + public final TopDocs topDocs() { + return new TopDocs(totalHits, docs); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + class SortFieldComparator implements Comparator { + + private FieldComparator [] comparators; + + public SortFieldComparator(int numHits, Sort sort) throws IOException { + SortField [] sortFlds = sort.getSort(); + int i = 0; + comparators = new FieldComparator[sortFlds.length]; + for (SortField fld : sortFlds) { + comparators[i++] = fld.getComparator(numHits, i); + } + } + + public int compare(ScoreDoc left, ScoreDoc right) { + int result = 0; + for (FieldComparator comp : comparators) { + result = comp.compare(left.doc, right.doc); + if (result != 0){ + break; + } + } + return result; + } + } +} Property changes on: src/java/org/apache/lucene/search/PostCollectSortCollector.java ___________________________________________________________________ Added: svn:eol-style + native Index: src/java/org/apache/lucene/search/Collector.java =================================================================== --- src/java/org/apache/lucene/search/Collector.java (revision 896062) +++ src/java/org/apache/lucene/search/Collector.java (working copy) @@ -143,7 +143,15 @@ */ public abstract void collect(int doc) throws IOException; + /** + * Called after all documents have been collected. Implementing classes can post process the + * results by overriding this method. For instance, in some instances, it may be worthwhile to postpone + * sorting until after the results are collected. + */ + public void postCollect() throws IOException {}; + + /** * Called before collecting from each IndexReader. All doc ids in * {@link #collect(int)} will correspond to reader. * Index: src/java/org/apache/lucene/search/MultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/MultiSearcher.java (revision 896062) +++ src/java/org/apache/lucene/search/MultiSearcher.java (working copy) @@ -269,6 +269,7 @@ searchables[i].search(weight, filter, hc); } + collector.postCollect(); } @Override Index: src/java/org/apache/lucene/search/ParallelMultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/ParallelMultiSearcher.java (revision 896062) +++ src/java/org/apache/lucene/search/ParallelMultiSearcher.java (working copy) @@ -174,6 +174,7 @@ searchables[i].search(weight, filter, hc); } + collector.postCollect(); } @Override Index: src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- src/java/org/apache/lucene/search/IndexSearcher.java (revision 896062) +++ src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -218,6 +218,7 @@ searchWithFilter(subReaders[i], weight, filter, collector); } } + collector.postCollect(); } private void searchWithFilter(IndexReader reader, Weight weight, Index: src/java/org/apache/lucene/util/ArrayUtil.java =================================================================== --- src/java/org/apache/lucene/util/ArrayUtil.java (revision 896062) +++ src/java/org/apache/lucene/util/ArrayUtil.java (working copy) @@ -1,5 +1,7 @@ package org.apache.lucene.util; +import org.apache.lucene.search.ScoreDoc; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -204,6 +206,20 @@ return grow(array, 1 + array.length); } + public static ScoreDoc[] grow(ScoreDoc[] array, int minSize) { + if (array.length < minSize) { + ScoreDoc[] newArray = new ScoreDoc[getNextSize(minSize)]; + System.arraycopy(array, 0, newArray, 0, array.length); + return newArray; + } else + return array; + } + + public static ScoreDoc[] grow(ScoreDoc[] array) { + return grow(array, 1 + array.length); + } + + public static byte[] shrink(byte[] array, int targetSize) { final int newSize = getShrinkSize(array.length, targetSize); if (newSize != array.length) { Index: contrib/benchmark/conf/collector.alg =================================================================== --- contrib/benchmark/conf/collector.alg (revision 0) +++ contrib/benchmark/conf/collector.alg (revision 0) @@ -0,0 +1,91 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# ------------------------------------------------------------------------------------- +# multi val params are iterated by NewRound's, added to reports, start with column name. + +# collector.class can be: +# Fully Qualified Class Name of a Collector with a empty constructor +# topDocOrdered - Creates a TopDocCollector that requires in order docs +# topDocUnordered - Like above, but allows out of order +collector.class=coll:topDocOrdered:topDocUnordered:topDocOrdered:topDocUnordered + +analyzer=org.apache.lucene.analysis.WhitespaceAnalyzer +directory=FSDirectory +#directory=RamDirectory + +doc.stored=true +doc.tokenized=true +doc.term.vector=false +log.step=100000 + +search.num.hits=1000000 + +content.source=org.apache.lucene.benchmark.byTask.feeds.LongToEnglishContentSource + + +query.maker=org.apache.lucene.benchmark.byTask.feeds.LongToEnglishQueryMaker + +# task at this depth or less would print when they start +task.max.depth.log=2 + +log.queries=true +# ------------------------------------------------------------------------------------- + +{ "Rounds" + + ResetSystemErase + + { "Populate" + CreateIndex + { "MAddDocs" AddDoc } : 2000000 + Optimize + CloseIndex + } + + OpenReader + { "topDocs" SearchWithCollector > : 10 + CloseReader + + OpenReader +#uses an array of search.num.hits size, but can also take in a parameter + { "psc" SearchWithPostSortCollector > : 10 + { "psc100" SearchWithPostSortCollector(100) > : 10 + { "psc1000" SearchWithPostSortCollector(1000) > : 10 + { "psc10000" SearchWithPostSortCollector(10000) > : 10 + { "psc50000" SearchWithPostSortCollector(50000) > : 10 + CloseReader + + RepSumByPref topDocs + RepSumByPref psc + RepSumByPref psc100 + RepSumByPref psc1000 + RepSumByPref psc10000 + RepSumByPref psc50000 + + NewRound + +} : 4 + +RepSumByNameRound +RepSumByName +RepSumByPrefRound topDocs +RepSumByPrefRound psc +RepSumByPrefRound psc100 +RepSumByPrefRound psc1000 +RepSumByPrefRound psc10000 +RepSumByPrefRound psc50000 + Index: contrib/benchmark/conf/collector-small.alg =================================================================== --- contrib/benchmark/conf/collector-small.alg (revision 0) +++ contrib/benchmark/conf/collector-small.alg (revision 0) @@ -0,0 +1,91 @@ +#/** +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# ------------------------------------------------------------------------------------- +# multi val params are iterated by NewRound's, added to reports, start with column name. + +# collector.class can be: +# Fully Qualified Class Name of a Collector with a empty constructor +# topDocOrdered - Creates a TopDocCollector that requires in order docs +# topDocUnordered - Like above, but allows out of order +collector.class=coll:topDocOrdered:topDocUnordered:topDocOrdered:topDocUnordered + +analyzer=org.apache.lucene.analysis.WhitespaceAnalyzer +directory=FSDirectory +#directory=RamDirectory + +doc.stored=true +doc.tokenized=true +doc.term.vector=false +log.step=100000 + +search.num.hits=100000 + +content.source=org.apache.lucene.benchmark.byTask.feeds.LongToEnglishContentSource + + +query.maker=org.apache.lucene.benchmark.byTask.feeds.LongToEnglishQueryMaker + +# task at this depth or less would print when they start +task.max.depth.log=2 + +log.queries=true +# ------------------------------------------------------------------------------------- + +{ "Rounds" + + ResetSystemErase + + { "Populate" + CreateIndex + { "MAddDocs" AddDoc } : 200000 + Optimize + CloseIndex + } + + OpenReader + { "topDocs" SearchWithCollector > : 10 + CloseReader + + OpenReader +#uses an array of search.num.hits size, but can also take in a parameter + { "psc" SearchWithPostSortCollector > : 10 + { "psc100" SearchWithPostSortCollector(100) > : 10 + { "psc1000" SearchWithPostSortCollector(1000) > : 10 + { "psc10000" SearchWithPostSortCollector(10000) > : 10 + { "psc50000" SearchWithPostSortCollector(50000) > : 10 + CloseReader + + RepSumByPref topDocs + RepSumByPref psc + RepSumByPref psc100 + RepSumByPref psc1000 + RepSumByPref psc10000 + RepSumByPref psc50000 + + NewRound + +} : 4 + +RepSumByNameRound +RepSumByName +RepSumByPrefRound topDocs +RepSumByPrefRound psc +RepSumByPrefRound psc100 +RepSumByPrefRound psc1000 +RepSumByPrefRound psc10000 +RepSumByPrefRound psc50000 + Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java (revision 896062) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/Config.java (working copy) @@ -22,20 +22,20 @@ import java.io.IOException; import java.io.Reader; import java.util.ArrayList; -import java.util.List; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Properties; import java.util.StringTokenizer; /** * Perf run configuration properties. - *

- * Numeric property containing ":", e.g. "10:100:5" is interpreted - * as array of numeric values. It is extracted once, on first use, and + *

+ * Numeric property containing ":", e.g. "10:100:5" is interpreted + * as array of numeric values. It is extracted once, on first use, and * maintain a round number to return the appropriate value. - *

- * The config property "work.dir" tells where is the root of + *

+ * The config property "work.dir" tells where is the root of * docs data dirs and indexes dirs. It is set to either of:

    *
  • value supplied for it in the alg file;
    • *
  • otherwise, value of System property "benchmark.work.dir";
    • @@ -48,30 +48,31 @@ private int roundNumber = 0; private Properties props; - private HashMap valByRound = new HashMap(); - private HashMap colForValByRound = new HashMap(); + private HashMap valByRound = new HashMap(); + private HashMap colForValByRound = new HashMap(); private String algorithmText; /** * Read both algorithm and config properties. + * * @param algReader from where to read algorithm and config properties. * @throws IOException */ - public Config (Reader algReader) throws IOException { + public Config(Reader algReader) throws IOException { // read alg file to array of lines ArrayList lines = new ArrayList(); BufferedReader r = new BufferedReader(algReader); - int lastConfigLine=0; - for (String line = r.readLine(); line!=null; line=r.readLine()) { + int lastConfigLine = 0; + for (String line = r.readLine(); line != null; line = r.readLine()) { lines.add(line); - if (line.indexOf('=')>0) { + if (line.indexOf('=') > 0) { lastConfigLine = lines.size(); } } r.close(); // copy props lines to string StringBuffer sb = new StringBuffer(); - for (int i=0; i Round ").append(roundNumber-1).append("-->").append(roundNumber); + StringBuffer sb = new StringBuffer("--> Round ").append(roundNumber - 1).append("-->").append(roundNumber); + // log changes in values - if (valByRound.size()>0) { + if (valByRound.size() > 0) { sb.append(": "); for (final String name : valByRound.keySet()) { Object a = valByRound.get(name); if (a instanceof int[]) { int ai[] = (int[]) a; - int n1 = (roundNumber-1)%ai.length; - int n2 = roundNumber%ai.length; + int n1 = (roundNumber - 1) % ai.length; + int n2 = roundNumber % ai.length; sb.append(" ").append(name).append(":").append(ai[n1]).append("-->").append(ai[n2]); - } else if (a instanceof double[]){ + } else if (a instanceof double[]) { double ad[] = (double[]) a; - int n1 = (roundNumber-1)%ad.length; - int n2 = roundNumber%ad.length; + int n1 = (roundNumber - 1) % ad.length; + int n2 = roundNumber % ad.length; sb.append(" ").append(name).append(":").append(ad[n1]).append("-->").append(ad[n2]); - } - else { + } else if (a instanceof String[]) { + String ad[] = (String[]) a; + int n1 = (roundNumber - 1) % ad.length; + int n2 = roundNumber % ad.length; + sb.append(" ").append(name).append(":").append(ad[n1]).append("-->").append(ad[n2]); + } else { boolean ab[] = (boolean[]) a; - int n1 = (roundNumber-1)%ab.length; - int n2 = roundNumber%ab.length; + int n1 = (roundNumber - 1) % ab.length; + int n2 = roundNumber % ab.length; sb.append(" ").append(name).append(":").append(ab[n1]).append("-->").append(ab[n2]); } } @@ -270,62 +301,76 @@ System.out.println(); System.out.println(sb.toString()); System.out.println(); - + return roundNumber; } - + + private String[] propToStringArray(String s) { + if (s.indexOf(":") < 0) { + return new String[]{s}; + } + + ArrayList a = new ArrayList(); + StringTokenizer st = new StringTokenizer(s, ":"); + while (st.hasMoreTokens()) { + String t = st.nextToken(); + a.add(t); + } + return (String[]) a.toArray(new String[a.size()]); + } + // extract properties to array, e.g. for "10:100:5" return int[]{10,100,5}. - private int[] propToIntArray (String s) { - if (s.indexOf(":")<0) { - return new int [] { Integer.parseInt(s) }; + private int[] propToIntArray(String s) { + if (s.indexOf(":") < 0) { + return new int[]{Integer.parseInt(s)}; } - + ArrayList a = new ArrayList(); - StringTokenizer st = new StringTokenizer(s,":"); + StringTokenizer st = new StringTokenizer(s, ":"); while (st.hasMoreTokens()) { String t = st.nextToken(); a.add(Integer.valueOf(t)); } - int res[] = new int[a.size()]; - for (int i=0; i a = new ArrayList(); - StringTokenizer st = new StringTokenizer(s,":"); + StringTokenizer st = new StringTokenizer(s, ":"); while (st.hasMoreTokens()) { String t = st.nextToken(); a.add(Double.valueOf(t)); } - double res[] = new double[a.size()]; - for (int i=0; i a = new ArrayList(); - StringTokenizer st = new StringTokenizer(s,":"); + StringTokenizer st = new StringTokenizer(s, ":"); while (st.hasMoreTokens()) { String t = st.nextToken(); a.add(new Boolean(t)); } - boolean res[] = new boolean[a.size()]; - for (int i=0; i 0"); + } + scoreDocs = new ScoreDoc[arraySize]; + } +} + Property changes on: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithPostSortCollectorTask.java ___________________________________________________________________ Added: svn:eol-style + native Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java (revision 0) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java (revision 0) @@ -0,0 +1,95 @@ +package org.apache.lucene.benchmark.byTask.tasks; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.benchmark.byTask.PerfRunData; +import org.apache.lucene.benchmark.byTask.feeds.QueryMaker; +import org.apache.lucene.benchmark.byTask.utils.Config; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.TopScoreDocCollector; + +import java.io.IOException; + +/** + * Does search w/ a custom collector + */ +public class SearchWithCollectorTask extends SearchTask { + + protected String clnName; + + public SearchWithCollectorTask(PerfRunData runData) { + super(runData); + } + + @Override + public void setup() throws Exception { + super.setup(); + //check to make sure either the doc is being stored + PerfRunData runData = getRunData(); + Config config = runData.getConfig(); + clnName = config.get("collector.class", ""); + } + + + + @Override + public boolean withCollector() { + return true; + } + + @Override + protected Collector createCollector() throws Exception { + Collector collector = null; + if (clnName.equalsIgnoreCase("topDocOrdered") == true) { + collector = TopScoreDocCollector.create(numHits(), true); + } else if (clnName.equalsIgnoreCase("topDocUnOrdered") == true) { + collector = TopScoreDocCollector.create(numHits(), false); + } else if (clnName.equals("") == false){ + Class clazz = (Class) Class.forName(clnName); + collector = clazz.newInstance(); + } else { + collector = super.createCollector(); + } + return collector; + } + + @Override + public QueryMaker getQueryMaker() { + return getRunData().getQueryMaker(this); + } + + @Override + public boolean withRetrieve() { + return false; + } + + @Override + public boolean withSearch() { + return true; + } + + @Override + public boolean withTraverse() { + return false; + } + + @Override + public boolean withWarm() { + return false; + } + +} Property changes on: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithCollectorTask.java ___________________________________________________________________ Added: svn:eol-style + native Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (revision 896062) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (working copy) @@ -30,10 +30,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.TopFieldCollector; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.Weight; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -105,23 +107,29 @@ res++; Query q = queryMaker.makeQuery(); Sort sort = getSort(); - TopDocs hits; + TopDocs hits = null; final int numHits = numHits(); if (numHits > 0) { - if (sort != null) { - Weight w = q.weight(searcher); - TopFieldCollector collector = TopFieldCollector.create(sort, numHits, - true, withScore(), - withMaxScore(), - !w.scoresDocsOutOfOrder()); - searcher.search(w, null, collector); - hits = collector.topDocs(); + if (withCollector() == false) { + if (sort != null) { + Weight w = q.weight(searcher); + TopFieldCollector collector = TopFieldCollector.create(sort, numHits, + true, withScore(), + withMaxScore(), + !w.scoresDocsOutOfOrder()); + searcher.search(w, null, collector); + hits = collector.topDocs(); + } else { + hits = searcher.search(q, numHits); + } } else { - hits = searcher.search(q, numHits); + Collector collector = createCollector(); + searcher.search(q, null, collector); + //hits = collector.topDocs(); } final String printHitsField = getRunData().getConfig().get("print.hits.field", null); - if (printHitsField != null && printHitsField.length() > 0) { + if (hits != null && printHitsField != null && printHitsField.length() > 0) { if (q instanceof MultiTermQuery) { System.out.println("MultiTermQuery term count = " + ((MultiTermQuery) q).getTotalNumberOfTerms()); } @@ -177,6 +185,9 @@ return res; } + protected Collector createCollector() throws Exception { + return TopScoreDocCollector.create(numHits(), true); + } protected Document retrieveDoc(IndexReader ir, int id) throws IOException { @@ -192,6 +203,10 @@ * Return true if search should be performed. */ public abstract boolean withSearch(); + + public boolean withCollector(){ + return false; + } /** Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java (revision 0) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java (revision 0) @@ -0,0 +1,49 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.benchmark.byTask.tasks.NewAnalyzerTask; +import org.apache.lucene.benchmark.byTask.utils.Config; +import org.apache.lucene.queryParser.QueryParser; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.English; +import org.apache.lucene.util.Version; + + +/** + * + * + **/ +public class LongToEnglishQueryMaker implements QueryMaker { + long counter = Long.MIN_VALUE + 10; + protected QueryParser parser; + + public Query makeQuery(int size) throws Exception { + throw new UnsupportedOperationException(); + } + + public synchronized Query makeQuery() throws Exception { + + return parser.parse("" + English.longToEnglish(getNextCounter()) + ""); + } + + private synchronized long getNextCounter() { + if (counter == Long.MAX_VALUE){ + counter = Long.MIN_VALUE + 10; + } + return counter++; + } + + public void setConfig(Config config) throws Exception { + Analyzer anlzr = NewAnalyzerTask.createAnalyzer(config.get("analyzer", StandardAnalyzer.class.getName())); + parser = new QueryParser(Version.LUCENE_CURRENT, DocMaker.BODY_FIELD, anlzr); + } + + public void resetInputs() { + counter = Long.MIN_VALUE + 10; + } + + public String printQueries() { + return "LongToEnglish: [" + Long.MIN_VALUE + " TO " + counter + "]"; + } +} Property changes on: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java ___________________________________________________________________ Added: svn:eol-style + native Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java (revision 0) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java (revision 0) @@ -0,0 +1,37 @@ +package org.apache.lucene.benchmark.byTask.feeds; + +import org.apache.lucene.util.English; + +import java.io.IOException; +import java.util.Date; + + +/** + * + * + **/ +public class LongToEnglishContentSource extends ContentSource{ + private long counter = Long.MIN_VALUE + 10; + + public void close() throws IOException { + + } + //TODO: reduce/clean up synchonization + public synchronized DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException { + docData.clear(); + docData.setBody(English.longToEnglish(counter)); + docData.setName("doc_" + String.valueOf(counter)); + docData.setTitle("title_" + String.valueOf(counter)); + docData.setDate(new Date()); + if (counter == Long.MAX_VALUE){ + counter = Long.MIN_VALUE + 10;//loop around + } + counter++; + return docData; + } + + @Override + public void resetInputs() throws IOException { + counter = Long.MIN_VALUE + 10; + } +} Property changes on: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java ___________________________________________________________________ Added: svn:eol-style + native Index: contrib/benchmark/build.xml =================================================================== --- contrib/benchmark/build.xml (revision 896062) +++ contrib/benchmark/build.xml (working copy) @@ -104,6 +104,7 @@ + @@ -120,7 +121,7 @@ - Working Directory: ${working.dir}