Index: src/test/org/apache/lucene/search/TestSort.java =================================================================== --- src/test/org/apache/lucene/search/TestSort.java (revision 728731) +++ src/test/org/apache/lucene/search/TestSort.java (working copy) @@ -24,9 +24,12 @@ import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.queryParser.ParseException; +import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.DocIdBitSet; @@ -39,6 +42,7 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Locale; +import java.util.Random; import java.util.regex.Pattern; /** @@ -54,469 +58,562 @@ extends TestCase implements Serializable { - private Searcher full; - private Searcher searchX; - private Searcher searchY; - private Query queryX; - private Query queryY; - private Query queryA; - private Query queryE; - private Query queryF; - private Query queryG; - private Sort sort; + private static final int NUM_STRINGS = 1000; + private Searcher full; + private Searcher searchX; + private Searcher searchY; + private Query queryX; + private Query queryY; + private Query queryA; + private Query queryE; + private Query queryF; + private Query queryG; + private Sort sort; - public TestSort (String name) { - super (name); - } + public TestSort (String name) { + super (name); + } - public static void main (String[] argv) { - if (argv == null || argv.length < 1) - TestRunner.run (suite()); - else if ("server".equals (argv[0])) { - TestSort test = new TestSort (null); - try { - test.startServer(); - Thread.sleep (500000); - } catch (Exception e) { - System.out.println (e); - e.printStackTrace(); - } - } - } + public static void main (String[] argv) { + if (argv == null || argv.length < 1) + TestRunner.run (suite()); + else if ("server".equals (argv[0])) { + TestSort test = new TestSort (null); + try { + test.startServer(); + Thread.sleep (500000); + } catch (Exception e) { + System.out.println (e); + e.printStackTrace(); + } + } + } - public static Test suite() { - return new TestSuite (TestSort.class); - } + public static Test suite() { + return new TestSuite (TestSort.class); + } - // document data: - // the tracer field is used to determine which document was hit - // the contents field is used to search and sort by relevance - // the int field to sort by int - // the float field to sort by float - // the string field to sort by string + // document data: + // the tracer field is used to determine which document was hit + // the contents field is used to search and sort by relevance + // the int field to sort by int + // the float field to sort by float + // the string field to sort by string // the i18n field includes accented characters for testing locale-specific sorting - private String[][] data = new String[][] { - // tracer contents int float string custom i18n long double, 'short', byte, 'custom parser encoding' - { "A", "x a", "5", "4f", "c", "A-3", "p\u00EAche", "10", "-4.0", "3", "126", "J"},//A, x - { "B", "y a", "5", "3.4028235E38", "i", "B-10", "HAT", "1000000000", "40.0", "24", "1", "I"},//B, y - { "C", "x a b c", "2147483647", "1.0", "j", "A-2", "p\u00E9ch\u00E9", "99999999", "40.00002343", "125", "15", "H"},//C, x - { "D", "y a b c", "-1", "0.0f", "a", "C-0", "HUT", String.valueOf(Long.MAX_VALUE), String.valueOf(Double.MIN_VALUE), String.valueOf(Short.MIN_VALUE), String.valueOf(Byte.MIN_VALUE), "G"},//D, y - { "E", "x a b c d", "5", "2f", "h", "B-8", "peach", String.valueOf(Long.MIN_VALUE), String.valueOf(Double.MAX_VALUE), String.valueOf(Short.MAX_VALUE), String.valueOf(Byte.MAX_VALUE), "F"},//E,x - { "F", "y a b c d", "2", "3.14159f", "g", "B-1", "H\u00C5T", "-44", "343.034435444", "-3", "0", "E"},//F,y - { "G", "x a b c d", "3", "-1.0", "f", "C-100", "sin", "323254543543", "4.043544", "5", "100", "D"},//G,x + private String[][] data = new String[][] { + // tracer contents int float string custom i18n long double, 'short', byte, 'custom parser encoding' + { "A", "x a", "5", "4f", "c", "A-3", "p\u00EAche", "10", "-4.0", "3", "126", "J"},//A, x + { "B", "y a", "5", "3.4028235E38", "i", "B-10", "HAT", "1000000000", "40.0", "24", "1", "I"},//B, y + { "C", "x a b c", "2147483647", "1.0", "j", "A-2", "p\u00E9ch\u00E9", "99999999", "40.00002343", "125", "15", "H"},//C, x + { "D", "y a b c", "-1", "0.0f", "a", "C-0", "HUT", String.valueOf(Long.MAX_VALUE), String.valueOf(Double.MIN_VALUE), String.valueOf(Short.MIN_VALUE), String.valueOf(Byte.MIN_VALUE), "G"},//D, y + { "E", "x a b c d", "5", "2f", "h", "B-8", "peach", String.valueOf(Long.MIN_VALUE), String.valueOf(Double.MAX_VALUE), String.valueOf(Short.MAX_VALUE), String.valueOf(Byte.MAX_VALUE), "F"},//E,x + { "F", "y a b c d", "2", "3.14159f", "g", "B-1", "H\u00C5T", "-44", "343.034435444", "-3", "0", "E"},//F,y + { "G", "x a b c d", "3", "-1.0", "f", "C-100", "sin", "323254543543", "4.043544", "5", "100", "D"},//G,x { "H", "y a b c d", "0", "1.4E-45", "e", "C-88", "H\u00D8T", "1023423423005","4.043545", "10", "-50", "C"},//H,y - { "I", "x a b c d e f", "-2147483648", "1.0e+0", "d", "A-10", "s\u00EDn", "332422459999", "4.043546", "-340", "51", "B"},//I,x - { "J", "y a b c d e f", "4", ".5", "b", "C-7", "HOT", "34334543543", "4.0000220343", "300", "2", "A"},//J,y - { "W", "g", "1", null, null, null, null, null, null, null, null, null}, - { "X", "g", "1", "0.1", null, null, null, null, null, null, null, null}, - { "Y", "g", "1", "0.2", null, null, null, null, null, null, null, null}, - { "Z", "f g", null, null, null, null, null, null, null, null, null, null} - }; + { "I", "x a b c d e f", "-2147483648", "1.0e+0", "d", "A-10", "s\u00EDn", "332422459999", "4.043546", "-340", "51", "B"},//I,x + { "J", "y a b c d e f", "4", ".5", "b", "C-7", "HOT", "34334543543", "4.0000220343", "300", "2", "A"},//J,y + { "W", "g", "1", null, null, null, null, null, null, null, null, null}, + { "X", "g", "1", "0.1", null, null, null, null, null, null, null, null}, + { "Y", "g", "1", "0.2", null, null, null, null, null, null, null, null}, + { "Z", "f g", null, null, null, null, null, null, null, null, null, null} + }; - // create an index of all the documents, or just the x, or just the y documents - private Searcher getIndex (boolean even, boolean odd) - throws IOException { - RAMDirectory indexStore = new RAMDirectory (); - IndexWriter writer = new IndexWriter (indexStore, new SimpleAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); - for (int i=0; i= 0; + if (!test) { + fail = true; + System.out.println("fail:" + v[j] + " " + last); + } + } + last = v[j]; + buff.append(v[j] + " "); + } + } + //System.out.println("buf:" + buff); + assertFalse(fail); + } - sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){ - public final short parseShort(final String val) { - return (short) (val.charAt(0)-'A'); - } - }), SortField.FIELD_DOC }); - assertMatches (full, queryA, sort, "JIHGFEDCBA"); - } + // test sorts where the type of field is specified and a custom field parser is used, that + // uses a simple char encoding. The sorted string contains a character beginning from 'A' that + // is mapped to a numeric value using some "funny" algorithm to be different for each data type. + public void testCustomFieldParserSort() throws Exception { + sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.IntParser(){ + public final int parseInt(final String val) { + return (int) (val.charAt(0)-'A') * 123456; + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); - // test sorts when there's nothing in the index - public void testEmptyIndex() throws Exception { - Searcher empty = getEmptyIndex(); + sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.FloatParser(){ + public final float parseFloat(final String val) { + return (float) Math.sqrt( (double) val.charAt(0) ); + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); - sort = new Sort(); - assertMatches (empty, queryX, sort, ""); + sort.setSort (new SortField[] { new SortField ("parser", new ExtendedFieldCache.LongParser(){ + public final long parseLong(final String val) { + return (long) (val.charAt(0)-'A') * 1234567890L; + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); - sort.setSort(SortField.FIELD_DOC); - assertMatches (empty, queryX, sort, ""); + sort.setSort (new SortField[] { new SortField ("parser", new ExtendedFieldCache.DoubleParser(){ + public final double parseDouble(final String val) { + return Math.pow( (double) val.charAt(0), (double) (val.charAt(0)-'A') ); + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); - sort.setSort (new SortField[] { new SortField ("int", SortField.INT), SortField.FIELD_DOC }); - assertMatches (empty, queryX, sort, ""); + sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ByteParser(){ + public final byte parseByte(final String val) { + return (byte) (val.charAt(0)-'A'); + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); - sort.setSort (new SortField[] { new SortField ("string", SortField.STRING, true), SortField.FIELD_DOC }); - assertMatches (empty, queryX, sort, ""); + sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){ + public final short parseShort(final String val) { + return (short) (val.charAt(0)-'A'); + } + }), SortField.FIELD_DOC }); + assertMatches (full, queryA, sort, "JIHGFEDCBA"); + } - sort.setSort (new SortField[] { new SortField ("float", SortField.FLOAT), new SortField ("string", SortField.STRING) }); - assertMatches (empty, queryX, sort, ""); - } + // test sorts when there's nothing in the index + public void testEmptyIndex() throws Exception { + Searcher empty = getEmptyIndex(); - // test sorts where the type of field is determined dynamically - public void testAutoSort() throws Exception { - sort.setSort("int"); - assertMatches (full, queryX, sort, "IGAEC"); - assertMatches (full, queryY, sort, "DHFJB"); + sort = new Sort(); + assertMatches (empty, queryX, sort, ""); - sort.setSort("float"); - assertMatches (full, queryX, sort, "GCIEA"); - assertMatches (full, queryY, sort, "DHJFB"); + sort.setSort(SortField.FIELD_DOC); + assertMatches (empty, queryX, sort, ""); - sort.setSort("string"); - assertMatches (full, queryX, sort, "AIGEC"); - assertMatches (full, queryY, sort, "DJHFB"); - } + sort.setSort (new SortField[] { new SortField ("int", SortField.INT), SortField.FIELD_DOC }); + assertMatches (empty, queryX, sort, ""); - // test sorts in reverse - public void testReverseSort() throws Exception { - sort.setSort (new SortField[] { new SortField (null, SortField.SCORE, true), SortField.FIELD_DOC }); - assertMatches (full, queryX, sort, "IEGCA"); - assertMatches (full, queryY, sort, "JFHDB"); + sort.setSort (new SortField[] { new SortField ("string", SortField.STRING, true), SortField.FIELD_DOC }); + assertMatches (empty, queryX, sort, ""); - sort.setSort (new SortField (null, SortField.DOC, true)); - assertMatches (full, queryX, sort, "IGECA"); - assertMatches (full, queryY, sort, "JHFDB"); + sort.setSort (new SortField[] { new SortField ("float", SortField.FLOAT), new SortField ("string", SortField.STRING) }); + assertMatches (empty, queryX, sort, ""); + } - sort.setSort ("int", true); - assertMatches (full, queryX, sort, "CAEGI"); - assertMatches (full, queryY, sort, "BJFHD"); + // test sorts where the type of field is determined dynamically + public void testAutoSort() throws Exception { + sort.setSort("int"); + assertMatches (full, queryX, sort, "IGAEC"); + assertMatches (full, queryY, sort, "DHFJB"); - sort.setSort ("float", true); - assertMatches (full, queryX, sort, "AECIG"); - assertMatches (full, queryY, sort, "BFJHD"); + sort.setSort("float"); + assertMatches (full, queryX, sort, "GCIEA"); + assertMatches (full, queryY, sort, "DHJFB"); - sort.setSort ("string", true); - assertMatches (full, queryX, sort, "CEGIA"); - assertMatches (full, queryY, sort, "BFHJD"); - } + sort.setSort("string"); + assertMatches (full, queryX, sort, "AIGEC"); + assertMatches (full, queryY, sort, "DJHFB"); + } - // test sorting when the sort field is empty (undefined) for some of the documents - public void testEmptyFieldSort() throws Exception { - sort.setSort ("string"); - assertMatches (full, queryF, sort, "ZJI"); + // test sorts in reverse + public void testReverseSort() throws Exception { + sort.setSort (new SortField[] { new SortField (null, SortField.SCORE, true), SortField.FIELD_DOC }); + assertMatches (full, queryX, sort, "IEGCA"); + assertMatches (full, queryY, sort, "JFHDB"); - sort.setSort ("string", true); - assertMatches (full, queryF, sort, "IJZ"); - - sort.setSort (new SortField ("i18n", Locale.ENGLISH)); - assertMatches (full, queryF, sort, "ZJI"); - - sort.setSort (new SortField ("i18n", Locale.ENGLISH, true)); - assertMatches (full, queryF, sort, "IJZ"); + sort.setSort (new SortField (null, SortField.DOC, true)); + assertMatches (full, queryX, sort, "IGECA"); + assertMatches (full, queryY, sort, "JHFDB"); - sort.setSort ("int"); - assertMatches (full, queryF, sort, "IZJ"); + sort.setSort ("int", true); + assertMatches (full, queryX, sort, "CAEGI"); + assertMatches (full, queryY, sort, "BJFHD"); - sort.setSort ("int", true); - assertMatches (full, queryF, sort, "JZI"); + sort.setSort ("float", true); + assertMatches (full, queryX, sort, "AECIG"); + assertMatches (full, queryY, sort, "BFJHD"); - sort.setSort ("float"); - assertMatches (full, queryF, sort, "ZJI"); + sort.setSort ("string", true); + assertMatches (full, queryX, sort, "CEGIA"); + assertMatches (full, queryY, sort, "BFHJD"); + } - // using a nonexisting field as first sort key shouldn't make a difference: - sort.setSort (new SortField[] { new SortField ("nosuchfield", SortField.STRING), - new SortField ("float") }); - assertMatches (full, queryF, sort, "ZJI"); + // test sorting when the sort field is empty (undefined) for some of the documents + public void testEmptyFieldSort() throws Exception { + sort.setSort ("string"); + assertMatches (full, queryF, sort, "ZJI"); - sort.setSort ("float", true); - assertMatches (full, queryF, sort, "IJZ"); + sort.setSort ("string", true); + assertMatches (full, queryF, sort, "IJZ"); + + sort.setSort (new SortField ("i18n", Locale.ENGLISH)); + assertMatches (full, queryF, sort, "ZJI"); + + sort.setSort (new SortField ("i18n", Locale.ENGLISH, true)); + assertMatches (full, queryF, sort, "IJZ"); - // When a field is null for both documents, the next SortField should be used. + sort.setSort ("int"); + assertMatches (full, queryF, sort, "IZJ"); + + sort.setSort ("int", true); + assertMatches (full, queryF, sort, "JZI"); + + sort.setSort ("float"); + assertMatches (full, queryF, sort, "ZJI"); + + // using a nonexisting field as first sort key shouldn't make a difference: + sort.setSort (new SortField[] { new SortField ("nosuchfield", SortField.STRING), + new SortField ("float") }); + assertMatches (full, queryF, sort, "ZJI"); + + sort.setSort ("float", true); + assertMatches (full, queryF, sort, "IJZ"); + + // When a field is null for both documents, the next SortField should be used. // Works for - sort.setSort (new SortField[] { new SortField ("int"), + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float") }); - assertMatches (full, queryG, sort, "ZWXY"); + new SortField ("float") }); + assertMatches (full, queryG, sort, "ZWXY"); - // Reverse the last criterium to make sure the test didn't pass by chance - sort.setSort (new SortField[] { new SortField ("int"), + // Reverse the last criterium to make sure the test didn't pass by chance + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float", true) }); - assertMatches (full, queryG, sort, "ZYXW"); + new SortField ("float", true) }); + assertMatches (full, queryG, sort, "ZYXW"); - // Do the same for a MultiSearcher - Searcher multiSearcher=new MultiSearcher (new Searchable[] { full }); + // Do the same for a MultiSearcher + Searcher multiSearcher=new MultiSearcher (new Searchable[] { full }); - sort.setSort (new SortField[] { new SortField ("int"), + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float") }); - assertMatches (multiSearcher, queryG, sort, "ZWXY"); + new SortField ("float") }); + assertMatches (multiSearcher, queryG, sort, "ZWXY"); - sort.setSort (new SortField[] { new SortField ("int"), + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float", true) }); - assertMatches (multiSearcher, queryG, sort, "ZYXW"); - // Don't close the multiSearcher. it would close the full searcher too! + new SortField ("float", true) }); + assertMatches (multiSearcher, queryG, sort, "ZYXW"); + // Don't close the multiSearcher. it would close the full searcher too! - // Do the same for a ParallelMultiSearcher + // Do the same for a ParallelMultiSearcher Searcher parallelSearcher=new ParallelMultiSearcher (new Searchable[] { full }); - sort.setSort (new SortField[] { new SortField ("int"), + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float") }); - assertMatches (parallelSearcher, queryG, sort, "ZWXY"); + new SortField ("float") }); + assertMatches (parallelSearcher, queryG, sort, "ZWXY"); - sort.setSort (new SortField[] { new SortField ("int"), + sort.setSort (new SortField[] { new SortField ("int"), new SortField ("string", SortField.STRING), - new SortField ("float", true) }); - assertMatches (parallelSearcher, queryG, sort, "ZYXW"); - // Don't close the parallelSearcher. it would close the full searcher too! - } + new SortField ("float", true) }); + assertMatches (parallelSearcher, queryG, sort, "ZYXW"); + // Don't close the parallelSearcher. it would close the full searcher too! + } - // test sorts using a series of fields - public void testSortCombos() throws Exception { - sort.setSort (new String[] {"int","float"}); - assertMatches (full, queryX, sort, "IGEAC"); + // test sorts using a series of fields + public void testSortCombos() throws Exception { + sort.setSort (new String[] {"int","float"}); + assertMatches (full, queryX, sort, "IGEAC"); - sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) }); - assertMatches (full, queryX, sort, "CEAGI"); + sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) }); + assertMatches (full, queryX, sort, "CEAGI"); - sort.setSort (new String[] {"float","string"}); - assertMatches (full, queryX, sort, "GICEA"); - } + sort.setSort (new String[] {"float","string"}); + assertMatches (full, queryX, sort, "GICEA"); + } - // test using a Locale for sorting strings - public void testLocaleSort() throws Exception { - sort.setSort (new SortField[] { new SortField ("string", Locale.US) }); - assertMatches (full, queryX, sort, "AIGEC"); - assertMatches (full, queryY, sort, "DJHFB"); + // test using a Locale for sorting strings + public void testLocaleSort() throws Exception { + sort.setSort (new SortField[] { new SortField ("string", Locale.US) }); + assertMatches (full, queryX, sort, "AIGEC"); + assertMatches (full, queryY, sort, "DJHFB"); - sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) }); - assertMatches (full, queryX, sort, "CEGIA"); - assertMatches (full, queryY, sort, "BFHJD"); - } + sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) }); + assertMatches (full, queryX, sort, "CEGIA"); + assertMatches (full, queryY, sort, "BFHJD"); + } - // test using various international locales with accented characters - // (which sort differently depending on locale) - public void testInternationalSort() throws Exception { - sort.setSort (new SortField ("i18n", Locale.US)); - assertMatches (full, queryY, sort, "BFJDH"); + // test using various international locales with accented characters + // (which sort differently depending on locale) + public void testInternationalSort() throws Exception { + sort.setSort (new SortField ("i18n", Locale.US)); + assertMatches (full, queryY, sort, "BFJDH"); - sort.setSort (new SortField ("i18n", new Locale("sv", "se"))); - assertMatches (full, queryY, sort, "BJDFH"); + sort.setSort (new SortField ("i18n", new Locale("sv", "se"))); + assertMatches (full, queryY, sort, "BJDFH"); - sort.setSort (new SortField ("i18n", new Locale("da", "dk"))); - assertMatches (full, queryY, sort, "BJDHF"); + sort.setSort (new SortField ("i18n", new Locale("da", "dk"))); + assertMatches (full, queryY, sort, "BJDHF"); - sort.setSort (new SortField ("i18n", Locale.US)); - assertMatches (full, queryX, sort, "ECAGI"); + sort.setSort (new SortField ("i18n", Locale.US)); + assertMatches (full, queryX, sort, "ECAGI"); - sort.setSort (new SortField ("i18n", Locale.FRANCE)); - assertMatches (full, queryX, sort, "EACGI"); - } + sort.setSort (new SortField ("i18n", Locale.FRANCE)); + assertMatches (full, queryX, sort, "EACGI"); + } // Test the MultiSearcher's ability to preserve locale-sensitive ordering // by wrapping it around a single searcher - public void testInternationalMultiSearcherSort() throws Exception { - Searcher multiSearcher = new MultiSearcher (new Searchable[] { full }); - - sort.setSort (new SortField ("i18n", new Locale("sv", "se"))); - assertMatches (multiSearcher, queryY, sort, "BJDFH"); - - sort.setSort (new SortField ("i18n", Locale.US)); - assertMatches (multiSearcher, queryY, sort, "BFJDH"); - - sort.setSort (new SortField ("i18n", new Locale("da", "dk"))); - assertMatches (multiSearcher, queryY, sort, "BJDHF"); - } + public void testInternationalMultiSearcherSort() throws Exception { + Searcher multiSearcher = new MultiSearcher (new Searchable[] { full }); - // test a custom sort function - public void testCustomSorts() throws Exception { - sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource())); - assertMatches (full, queryX, sort, "CAIEG"); - sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true)); - assertMatches (full, queryY, sort, "HJDBF"); - SortComparator custom = SampleComparable.getComparator(); - sort.setSort (new SortField ("custom", custom)); - assertMatches (full, queryX, sort, "CAIEG"); - sort.setSort (new SortField ("custom", custom, true)); - assertMatches (full, queryY, sort, "HJDBF"); - } + sort.setSort (new SortField ("i18n", new Locale("sv", "se"))); + assertMatches (multiSearcher, queryY, sort, "BJDFH"); + + sort.setSort (new SortField ("i18n", Locale.US)); + assertMatches (multiSearcher, queryY, sort, "BFJDH"); + + sort.setSort (new SortField ("i18n", new Locale("da", "dk"))); + assertMatches (multiSearcher, queryY, sort, "BJDHF"); + } + + // test a custom sort function + public void testCustomSorts() throws Exception { + sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource())); + assertMatches (full, queryX, sort, "CAIEG"); + sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true)); + assertMatches (full, queryY, sort, "HJDBF"); + SortComparator custom = SampleComparable.getComparator(); + sort.setSort (new SortField ("custom", custom)); + assertMatches (full, queryX, sort, "CAIEG"); + sort.setSort (new SortField ("custom", custom, true)); + assertMatches (full, queryY, sort, "HJDBF"); + } - // test a variety of sorts using more than one searcher - public void testMultiSort() throws Exception { - MultiSearcher searcher = new MultiSearcher (new Searchable[] { searchX, searchY }); - runMultiSorts (searcher); - } + // test a variety of sorts using more than one searcher + public void testMultiSort() throws Exception { + MultiSearcher searcher = new MultiSearcher (new Searchable[] { searchX, searchY }); + runMultiSorts (searcher); + } - // test a variety of sorts using a parallel multisearcher - public void testParallelMultiSort() throws Exception { - Searcher searcher = new ParallelMultiSearcher (new Searchable[] { searchX, searchY }); - runMultiSorts (searcher); - } + // test a variety of sorts using a parallel multisearcher + public void testParallelMultiSort() throws Exception { + Searcher searcher = new ParallelMultiSearcher (new Searchable[] { searchX, searchY }); + runMultiSorts (searcher); + } - // test a variety of sorts using a remote searcher - public void testRemoteSort() throws Exception { - Searchable searcher = getRemote(); - MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher }); - runMultiSorts (multi); - } + // test a variety of sorts using a remote searcher + public void testRemoteSort() throws Exception { + Searchable searcher = getRemote(); + MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher }); + runMultiSorts (multi); + } - // test custom search when remote - public void testRemoteCustomSort() throws Exception { - Searchable searcher = getRemote(); - MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher }); - sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource())); - assertMatches (multi, queryX, sort, "CAIEG"); - sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true)); - assertMatches (multi, queryY, sort, "HJDBF"); - SortComparator custom = SampleComparable.getComparator(); - sort.setSort (new SortField ("custom", custom)); - assertMatches (multi, queryX, sort, "CAIEG"); - sort.setSort (new SortField ("custom", custom, true)); - assertMatches (multi, queryY, sort, "HJDBF"); - } + // test custom search when remote + public void testRemoteCustomSort() throws Exception { + Searchable searcher = getRemote(); + MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher }); + sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource())); + assertMatches (multi, queryX, sort, "CAIEG"); + sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true)); + assertMatches (multi, queryY, sort, "HJDBF"); + SortComparator custom = SampleComparable.getComparator(); + sort.setSort (new SortField ("custom", custom)); + assertMatches (multi, queryX, sort, "CAIEG"); + sort.setSort (new SortField ("custom", custom, true)); + assertMatches (multi, queryY, sort, "HJDBF"); + } - // test that the relevancy scores are the same even if - // hits are sorted - public void testNormalizedScores() throws Exception { + // test that the relevancy scores are the same even if + // hits are sorted + public void testNormalizedScores() throws Exception { - // capture relevancy scores + // capture relevancy scores HashMap scoresX = getScores (full.search (queryX, null, 1000).scoreDocs, full); HashMap scoresY = getScores (full.search (queryY, null, 1000).scoreDocs, full); HashMap scoresA = getScores (full.search (queryA, null, 1000).scoreDocs, full); - // we'll test searching locally, remote and multi - MultiSearcher remote = new MultiSearcher (new Searchable[] { getRemote() }); - MultiSearcher multi = new MultiSearcher (new Searchable[] { searchX, searchY }); + // we'll test searching locally, remote and multi + MultiSearcher remote = new MultiSearcher (new Searchable[] { getRemote() }); + MultiSearcher multi = new MultiSearcher (new Searchable[] { searchX, searchY }); - // change sorting and make sure relevancy stays the same + // change sorting and make sure relevancy stays the same - sort = new Sort(); + sort = new Sort(); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -527,7 +624,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort(SortField.FIELD_DOC); + sort.setSort(SortField.FIELD_DOC); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -538,7 +635,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort ("int"); + sort.setSort ("int"); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -549,7 +646,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort ("float"); + sort.setSort ("float"); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -560,7 +657,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort ("string"); + sort.setSort ("string"); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -571,7 +668,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort (new String[] {"int","float"}); + sort.setSort (new String[] {"int","float"}); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -582,7 +679,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) }); + sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) }); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -593,7 +690,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - sort.setSort (new String[] {"float","string"}); + sort.setSort (new String[] {"float","string"}); assertSameValues (scoresX, getScores (full.search (queryX, null, 1000, sort).scoreDocs, full)); assertSameValues (scoresX, getScores (remote.search (queryX, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresX, getScores (multi.search (queryX, null, 1000, sort).scoreDocs, multi)); @@ -604,7 +701,7 @@ assertSameValues (scoresA, getScores (remote.search (queryA, null, 1000, sort).scoreDocs, remote)); assertSameValues (scoresA, getScores (multi.search (queryA, null, 1000, sort).scoreDocs, multi)); - } + } public void testTopDocsScores() throws Exception { @@ -621,6 +718,7 @@ Filter filt = new Filter() { public DocIdSet getDocIdSet(IndexReader reader) throws IOException { BitSet bs = new BitSet(reader.maxDoc()); + bs.set(0, reader.maxDoc()); bs.set(docs1.scoreDocs[0].doc); return new DocIdBitSet(bs); } @@ -631,121 +729,120 @@ assertEquals(docs1.scoreDocs[0].score, docs2.scoreDocs[0].score, 1e-6); } - // runs a variety of sorts useful for multisearchers - private void runMultiSorts (Searcher multi) throws Exception { - sort.setSort (SortField.FIELD_DOC); - assertMatchesPattern (multi, queryA, sort, "[AB]{2}[CD]{2}[EF]{2}[GH]{2}[IJ]{2}"); + private void runMultiSorts (Searcher multi) throws Exception { + sort.setSort (SortField.FIELD_DOC); + assertMatchesPattern (multi, queryA, sort, "[AB]{2}[CD]{2}[EF]{2}[GH]{2}[IJ]{2}"); - sort.setSort (new SortField ("int", SortField.INT)); - assertMatchesPattern (multi, queryA, sort, "IDHFGJ[ABE]{3}C"); + sort.setSort (new SortField ("int", SortField.INT)); + assertMatchesPattern (multi, queryA, sort, "IDHFGJ[ABE]{3}C"); - sort.setSort (new SortField[] {new SortField ("int", SortField.INT), SortField.FIELD_DOC}); - assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC"); + sort.setSort (new SortField[] {new SortField ("int", SortField.INT), SortField.FIELD_DOC}); + assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC"); - sort.setSort ("int"); - assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC"); + sort.setSort ("int"); + assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC"); - sort.setSort (new SortField[] {new SortField ("float", SortField.FLOAT), SortField.FIELD_DOC}); - assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB"); + sort.setSort (new SortField[] {new SortField ("float", SortField.FLOAT), SortField.FIELD_DOC}); + assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB"); - sort.setSort ("float"); - assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB"); + sort.setSort ("float"); + assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB"); - sort.setSort ("string"); - assertMatches (multi, queryA, sort, "DJAIHGFEBC"); + sort.setSort ("string"); + assertMatches (multi, queryA, sort, "DJAIHGFEBC"); - sort.setSort ("int", true); - assertMatchesPattern (multi, queryA, sort, "C[AB]{2}EJGFHDI"); + sort.setSort ("int", true); + assertMatchesPattern (multi, queryA, sort, "C[AB]{2}EJGFHDI"); - sort.setSort ("float", true); - assertMatchesPattern (multi, queryA, sort, "BAFE[IC]{2}JHDG"); + sort.setSort ("float", true); + assertMatchesPattern (multi, queryA, sort, "BAFE[IC]{2}JHDG"); - sort.setSort ("string", true); - assertMatches (multi, queryA, sort, "CBEFGHIAJD"); + sort.setSort ("string", true); + assertMatches (multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort (new SortField[] { new SortField ("string", Locale.US) }); - assertMatches (multi, queryA, sort, "DJAIHGFEBC"); + sort.setSort (new SortField[] { new SortField ("string", Locale.US) }); + assertMatches (multi, queryA, sort, "DJAIHGFEBC"); - sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) }); - assertMatches (multi, queryA, sort, "CBEFGHIAJD"); + sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) }); + assertMatches (multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort (new String[] {"int","float"}); - assertMatches (multi, queryA, sort, "IDHFGJEABC"); + sort.setSort (new String[] {"int","float"}); + assertMatches (multi, queryA, sort, "IDHFGJEABC"); - sort.setSort (new String[] {"float","string"}); - assertMatches (multi, queryA, sort, "GDHJICEFAB"); + sort.setSort (new String[] {"float","string"}); + assertMatches (multi, queryA, sort, "GDHJICEFAB"); - sort.setSort ("int"); - assertMatches (multi, queryF, sort, "IZJ"); + sort.setSort ("int"); + assertMatches (multi, queryF, sort, "IZJ"); - sort.setSort ("int", true); - assertMatches (multi, queryF, sort, "JZI"); + sort.setSort ("int", true); + assertMatches (multi, queryF, sort, "JZI"); - sort.setSort ("float"); - assertMatches (multi, queryF, sort, "ZJI"); + sort.setSort ("float"); + assertMatches (multi, queryF, sort, "ZJI"); - sort.setSort ("string"); - assertMatches (multi, queryF, sort, "ZJI"); + sort.setSort ("string"); + assertMatches (multi, queryF, sort, "ZJI"); - sort.setSort ("string", true); - assertMatches (multi, queryF, sort, "IJZ"); - } + sort.setSort ("string", true); + assertMatches (multi, queryF, sort, "IJZ"); + } - // make sure the documents returned by the search match the expected list - private void assertMatches (Searcher searcher, Query query, Sort sort, String expectedResult) - throws IOException { + // make sure the documents returned by the search match the expected list + private void assertMatches (Searcher searcher, Query query, Sort sort, String expectedResult) + throws IOException { ScoreDoc[] result = searcher.search (query, null, 1000, sort).scoreDocs; - StringBuffer buff = new StringBuffer(10); + StringBuffer buff = new StringBuffer(10); int n = result.length; - for (int i=0; i 0 ) { try { Thread.sleep(slowdown); @@ -315,8 +320,8 @@ System.out.println("caught " + x); } } - bits.set( doc ); - lastDocCollected = doc; + bits.set( docId ); + lastDocCollected = docId; } public int hitCount() { @@ -326,6 +331,11 @@ public int getLastDocCollected() { return lastDocCollected; } + + public void setNextReader(IndexReader reader) { + docBase = (docBase == -1) ? 0 : docBase + lastMax; + lastMax = reader.maxDoc(); + } } Index: src/test/org/apache/lucene/search/TestTermScorer.java =================================================================== --- src/test/org/apache/lucene/search/TestTermScorer.java (revision 728731) +++ src/test/org/apache/lucene/search/TestTermScorer.java (working copy) @@ -82,15 +82,21 @@ //must call next first - ts.score(new HitCollector() + ts.score(new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public void collect(int doc, float score) { - docs.add(new TestHit(doc, score)); + docs.add(new TestHit(doc + base, score)); assertTrue("score " + score + " is not greater than 0", score > 0); assertTrue("Doc: " + doc + " does not equal: " + 0 + " or doc does not equaal: " + 5, doc == 0 || doc == 5); } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); assertTrue("docs Size: " + docs.size() + " is not: " + 2, docs.size() == 2); TestHit doc0 = (TestHit) docs.get(0); Index: src/test/org/apache/lucene/search/TestSetNorm.java =================================================================== --- src/test/org/apache/lucene/search/TestSetNorm.java (revision 728731) +++ src/test/org/apache/lucene/search/TestSetNorm.java (working copy) @@ -62,10 +62,16 @@ new IndexSearcher(store).search (new TermQuery(new Term("field", "word")), - new HitCollector() { + new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public final void collect(int doc, float score) { - scores[doc] = score; + scores[doc + base] = score; } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); float lastScore = 0.0f; Index: src/test/org/apache/lucene/search/TestScorerPerf.java =================================================================== --- src/test/org/apache/lucene/search/TestScorerPerf.java (revision 728731) +++ src/test/org/apache/lucene/search/TestScorerPerf.java (working copy) @@ -96,9 +96,11 @@ return sets; } - public static class CountingHitCollector extends HitCollector { + public static class CountingHitCollector extends MultiReaderHitCollector { int count=0; int sum=0; + protected int docBase = -1; + private int lastMax; public void collect(int doc, float score) { count++; @@ -107,6 +109,11 @@ public int getCount() { return count; } public int getSum() { return sum; } + + public void setNextReader(IndexReader reader) { + docBase = (docBase == -1) ? 0 : docBase + lastMax; + lastMax = reader.maxDoc(); + } } @@ -119,10 +126,10 @@ public void collect(int doc, float score) { pos = answer.nextSetBit(pos+1); - if (pos != doc) { - throw new RuntimeException("Expected doc " + pos + " but got " + doc); + if (pos != doc + docBase) { + throw new RuntimeException("Expected doc " + pos + " but got " + doc + docBase); } - super.collect(doc,score); + super.collect(doc+docBase,score); } } Index: src/test/org/apache/lucene/search/TestSimilarity.java =================================================================== --- src/test/org/apache/lucene/search/TestSimilarity.java (revision 728731) +++ src/test/org/apache/lucene/search/TestSimilarity.java (working copy) @@ -21,6 +21,7 @@ import java.util.Collection; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; @@ -75,10 +76,11 @@ searcher.search (new TermQuery(b), - new HitCollector() { + new MultiReaderHitCollector() { public final void collect(int doc, float score) { assertTrue(score == 1.0f); } + public void setNextReader(IndexReader reader) {} }); BooleanQuery bq = new BooleanQuery(); @@ -87,11 +89,17 @@ //System.out.println(bq.toString("field")); searcher.search (bq, - new HitCollector() { + new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public final void collect(int doc, float score) { //System.out.println("Doc=" + doc + " score=" + score); - assertTrue(score == (float)doc+1); + assertTrue(score == (float)doc+base+1); } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); PhraseQuery pq = new PhraseQuery(); @@ -100,22 +108,24 @@ //System.out.println(pq.toString("field")); searcher.search (pq, - new HitCollector() { + new MultiReaderHitCollector() { public final void collect(int doc, float score) { //System.out.println("Doc=" + doc + " score=" + score); assertTrue(score == 1.0f); } + public void setNextReader(IndexReader reader) {} }); pq.setSlop(2); //System.out.println(pq.toString("field")); searcher.search (pq, - new HitCollector() { + new MultiReaderHitCollector() { public final void collect(int doc, float score) { //System.out.println("Doc=" + doc + " score=" + score); assertTrue(score == 2.0f); } + public void setNextReader(IndexReader reader) {} }); } } Index: src/test/org/apache/lucene/search/TestDocBoost.java =================================================================== --- src/test/org/apache/lucene/search/TestDocBoost.java (revision 728731) +++ src/test/org/apache/lucene/search/TestDocBoost.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.RAMDirectory; @@ -65,10 +66,16 @@ new IndexSearcher(store).search (new TermQuery(new Term("field", "word")), - new HitCollector() { + new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public final void collect(int doc, float score) { - scores[doc] = score; + scores[doc + base] = score; } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); float lastScore = 0.0f; Index: src/test/org/apache/lucene/search/QueryUtils.java =================================================================== --- src/test/org/apache/lucene/search/QueryUtils.java (revision 728731) +++ src/test/org/apache/lucene/search/QueryUtils.java (working copy) @@ -8,6 +8,8 @@ import java.io.ObjectInputStream; import java.io.ObjectOutputStream; +import org.apache.lucene.index.IndexReader; + /** * Copyright 2005 Apache Software Foundation * @@ -151,8 +153,11 @@ final int[] sdoc = new int[] {-1}; final float maxDiff = 1e-5f; - s.search(q,new HitCollector() { + s.search(q,new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public void collect(int doc, float score) { + doc = doc + base; try { int op = order[(opidx[0]++)%order.length]; //System.out.println(op==skip_op ? "skip("+(sdoc[0]+1)+")":"next()"); @@ -183,6 +188,10 @@ throw new RuntimeException(e); } } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); // make sure next call to scorer is false. @@ -198,9 +207,12 @@ //System.out.println("checkFirstSkipTo: "+q); final float maxDiff = 1e-5f; final int lastDoc[] = {-1}; - s.search(q,new HitCollector() { + s.search(q,new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public void collect(int doc, float score) { //System.out.println("doc="+doc); + doc = doc + base; try { for (int i=lastDoc[0]+1; i<=doc; i++) { Weight w = q.weight(s); @@ -216,6 +228,10 @@ throw new RuntimeException(e); } } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); Weight w = q.weight(s); Scorer scorer = w.scorer(s.getIndexReader()); Index: src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- src/test/org/apache/lucene/index/TestOmitTf.java (revision 728731) +++ src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -26,8 +26,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.HitCollector; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiReaderHitCollector; import org.apache.lucene.search.Searcher; import org.apache.lucene.search.Similarity; import org.apache.lucene.search.TermQuery; @@ -348,16 +348,23 @@ dir.close(); } - public static class CountingHitCollector extends HitCollector { + public static class CountingHitCollector extends MultiReaderHitCollector { static int count=0; static int sum=0; + private int docBase = -1; + private int lastMax; CountingHitCollector(){count=0;sum=0;} public void collect(int doc, float score) { count++; - sum += doc; // use it to avoid any possibility of being optimized away + sum += doc + docBase; // use it to avoid any possibility of being optimized away } public static int getCount() { return count; } public static int getSum() { return sum; } + + public void setNextReader(IndexReader reader) { + docBase = (docBase == -1) ? 0 : docBase + lastMax; + lastMax = reader.maxDoc(); + } } } Index: src/test/org/apache/lucene/index/TestIndexReaderReopen.java =================================================================== --- src/test/org/apache/lucene/index/TestIndexReaderReopen.java (revision 728731) +++ src/test/org/apache/lucene/index/TestIndexReaderReopen.java (working copy) @@ -320,7 +320,7 @@ assertRefCountEquals(1, reader0); assertTrue(reader0 instanceof MultiSegmentReader); - SegmentReader[] subReaders0 = ((MultiSegmentReader) reader0).getSubReaders(); + IndexReader[] subReaders0 = ((MultiSegmentReader) reader0).getSequentialReaders(); for (int i = 0; i < subReaders0.length; i++) { assertRefCountEquals(1, subReaders0[i]); } @@ -332,7 +332,7 @@ IndexReader reader1 = refreshReader(reader0, true).refreshedReader; assertTrue(reader1 instanceof MultiSegmentReader); - SegmentReader[] subReaders1 = ((MultiSegmentReader) reader1).getSubReaders(); + IndexReader[] subReaders1 = ((MultiSegmentReader) reader1).getSequentialReaders(); assertEquals(subReaders0.length, subReaders1.length); for (int i = 0; i < subReaders0.length; i++) { @@ -349,7 +349,7 @@ IndexReader reader2 = refreshReader(reader1, true).refreshedReader; assertTrue(reader2 instanceof MultiSegmentReader); - SegmentReader[] subReaders2 = ((MultiSegmentReader) reader2).getSubReaders(); + IndexReader[] subReaders2 = ((MultiSegmentReader) reader2).getSequentialReaders(); assertEquals(subReaders1.length, subReaders2.length); for (int i = 0; i < subReaders2.length; i++) { @@ -373,7 +373,7 @@ IndexReader reader3 = refreshReader(reader0, true).refreshedReader; assertTrue(reader3 instanceof MultiSegmentReader); - SegmentReader[] subReaders3 = ((MultiSegmentReader) reader3).getSubReaders(); + IndexReader[] subReaders3 = ((MultiSegmentReader) reader3).getSequentialReaders(); assertEquals(subReaders3.length, subReaders0.length); // try some permutations @@ -945,14 +945,14 @@ if (checkSubReaders) { if (reader instanceof MultiSegmentReader) { - SegmentReader[] subReaders = ((MultiSegmentReader) reader).getSubReaders(); + IndexReader[] subReaders = ((MultiSegmentReader) reader).getSequentialReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); } } if (reader instanceof MultiReader) { - IndexReader[] subReaders = ((MultiReader) reader).getSubReaders(); + IndexReader[] subReaders = ((MultiReader) reader).getSequentialReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed); } @@ -971,7 +971,7 @@ reader.ensureOpen(); if (reader instanceof MultiSegmentReader) { - SegmentReader[] subReaders = ((MultiSegmentReader) reader).getSubReaders(); + IndexReader[] subReaders = ((MultiSegmentReader) reader).getSequentialReaders(); for (int i = 0; i < subReaders.length; i++) { assertReaderOpen(subReaders[i]); } Index: src/java/org/apache/lucene/search/TopFieldDocCollector.java =================================================================== --- src/java/org/apache/lucene/search/TopFieldDocCollector.java (revision 728731) +++ src/java/org/apache/lucene/search/TopFieldDocCollector.java (working copy) @@ -28,6 +28,7 @@ *

This may be extended, overriding the collect method to, e.g., * conditionally invoke super() in order to filter which * documents are collected. + * **/ public class TopFieldDocCollector extends TopDocCollector { @@ -48,7 +49,7 @@ if (score > 0.0f) { totalHits++; if (reusableFD == null) - reusableFD = new FieldDoc(doc, score); + reusableFD = new FieldDoc(doc + docBase, score); else { // Whereas TopDocCollector can skip this if the // score is not competitive, we cannot because the @@ -56,7 +57,7 @@ // aren't in general congruent with "higher score // wins" reusableFD.score = score; - reusableFD.doc = doc; + reusableFD.doc = doc + docBase; } reusableFD = (FieldDoc) hq.insertWithOverflow(reusableFD); } @@ -72,4 +73,5 @@ return new TopFieldDocs(totalHits, scoreDocs, fshq.getFields(), fshq.getMaxScore()); } + } Index: src/java/org/apache/lucene/search/ScoreDocComparator.java =================================================================== --- src/java/org/apache/lucene/search/ScoreDocComparator.java (revision 728731) +++ src/java/org/apache/lucene/search/ScoreDocComparator.java (working copy) @@ -25,6 +25,7 @@ * * @since lucene 1.4 * @version $Id$ + * @deprecated use {@link FieldComparator} */ public interface ScoreDocComparator { Index: src/java/org/apache/lucene/search/FieldValueHitQueue.java =================================================================== --- src/java/org/apache/lucene/search/FieldValueHitQueue.java (revision 0) +++ src/java/org/apache/lucene/search/FieldValueHitQueue.java (revision 0) @@ -0,0 +1,269 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermEnum; +import org.apache.lucene.util.PriorityQueue; + +import java.io.IOException; +import java.util.Locale; + +/** + * Expert: A hit queue for sorting by hits by terms in more than one field. + * Uses FieldCache.DEFAULT for maintaining + * internal term lookup tables. + * + * NOTE: This API is experimental and might change in + * incompatible ways in the next release. + * + * @since lucene 2.9 + * @version $Id: + * @see Searcher#search(Query,Filter,int,Sort) + * @see FieldCache + */ +public class FieldValueHitQueue extends PriorityQueue { + + final static class Entry { + int slot; + int subReaderIndex; + int subDocID; + float score; + Entry(int slot, int subReaderIndex, int subDocID, float score) { + this.slot = slot; + this.subReaderIndex = subReaderIndex; + this.subDocID = subDocID; + this.score = score; + } + + public String toString() { + return "slot:" + slot + " subR:" + subReaderIndex + " subid:" + subDocID; + } + } + + // nocommit + private static boolean first = true; + + /** + * Creates a hit queue sorted by the given list of fields. + * @param reader + * @param fields Fieldable names, in priority order (highest priority first). Cannot be null or empty. + * @param size The number of hits to retain. Must be greater than zero. + * @throws IOException + */ + public FieldValueHitQueue(SortField[] fields, int size) throws IOException { + numComparators = fields.length; + comparators = new FieldComparator[numComparators]; + reverseMul = new int[numComparators]; + + this.fields = fields; + for (int i=0; iThis may be extended, overriding the collect method to, e.g., * conditionally invoke super() in order to filter which * documents are collected. + * **/ -public class TopDocCollector extends HitCollector { +public class TopDocCollector extends MultiReaderHitCollector { private ScoreDoc reusableSD; @@ -36,6 +38,10 @@ /** The priority queue which holds the top-scoring documents. */ protected PriorityQueue hq; + + protected int docBase = -1; + + private int lastMax; /** Construct to collect a given number of hits. * @param numHits the maximum number of hits to collect @@ -63,12 +69,12 @@ if (score > 0.0f) { totalHits++; if (reusableSD == null) { - reusableSD = new ScoreDoc(doc, score); + reusableSD = new ScoreDoc(doc + docBase, score); } else if (score >= reusableSD.score) { // reusableSD holds the last "rejected" entry, so, if // this new score is not better than that, there's no // need to try inserting it - reusableSD.doc = doc; + reusableSD.doc = doc + docBase; reusableSD.score = score; } else { return; @@ -92,4 +98,9 @@ return new TopDocs(totalHits, scoreDocs, maxScore); } + + public void setNextReader(IndexReader reader) { + docBase = (docBase == -1) ? 0 : docBase + lastMax; + lastMax = reader.maxDoc(); + } } Index: src/java/org/apache/lucene/search/MultiReaderHitCollector.java =================================================================== --- src/java/org/apache/lucene/search/MultiReaderHitCollector.java (revision 0) +++ src/java/org/apache/lucene/search/MultiReaderHitCollector.java (revision 0) @@ -0,0 +1,50 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; + +/** + * Expert: A HitCollector that can be used to collect hits + * across sequential IndexReaders. For a Multi*Reader, this + * collector advances through each of the sub readers. This + * can be a higher performance means of collection. + * + * NOTE: The doc that is passed to the collect method + * is relative to the current reader. You must re-base the + * doc, by tracking the cumulative maxDoc() of each + * subreader, to map it to the docID space of the + * Multi*Reader. + * + * NOTE: This API is experimental and might change in + * incompatible ways in the next release. + */ +public abstract class MultiReaderHitCollector extends HitCollector { + + /** + * Called before collecting from each IndexReader. All doc + * ids in {@link #collect(int, float)} will correspond to reader. + * + * @param reader + * @throws IOException + */ + public abstract void setNextReader(IndexReader reader) throws IOException; + +} Property changes on: src/java/org/apache/lucene/search/MultiReaderHitCollector.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/search/HitCollector.java =================================================================== --- src/java/org/apache/lucene/search/HitCollector.java (revision 728731) +++ src/java/org/apache/lucene/search/HitCollector.java (working copy) @@ -19,7 +19,9 @@ /** Lower-level search API. *
HitCollectors are primarily meant to be used to implement queries, - * sorting and filtering. + * sorting and filtering. See {@link + * MultiReaderHitCollector} for a lower level and + * higher performance (on a multi-segment index) API. * @see Searcher#search(Query,HitCollector) * @version $Id$ */ Index: src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java =================================================================== --- src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java (revision 728731) +++ src/java/org/apache/lucene/search/ExtendedFieldCacheImpl.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.TermEnum; +import org.apache.lucene.search.FieldCacheImpl.Entry; import java.io.IOException; @@ -178,5 +179,17 @@ } } }; + + public Comparable[] getCustom(IndexReader reader, String field, + FieldComparator comparator) { + Comparable[] comps = null; + try { + comps = (Comparable[]) customCache.get(reader, new Entry(field, comparator)); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return comps; + } } Index: src/java/org/apache/lucene/search/FieldComparator.java =================================================================== --- src/java/org/apache/lucene/search/FieldComparator.java (revision 0) +++ src/java/org/apache/lucene/search/FieldComparator.java (revision 0) @@ -0,0 +1,1063 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.text.Collator; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.ExtendedFieldCache.DoubleParser; +import org.apache.lucene.search.ExtendedFieldCache.LongParser; +import org.apache.lucene.search.FieldCache.ByteParser; +import org.apache.lucene.search.FieldCache.FloatParser; +import org.apache.lucene.search.FieldCache.IntParser; +import org.apache.lucene.search.FieldCache.ShortParser; +import org.apache.lucene.search.FieldCache.StringIndex; + +/** + * A FieldComparator compares hits across multiple IndexReaders. + * + * A comparator can compare a hit at hit 'slot a' with hit 'slot b', + * compare a hit on 'doc i' with hit 'slot a', or copy a hit at 'doc i' + * to 'slot a'. Each slot refers to a hit while each doc refers to the + * current IndexReader. + * + * NOTE: This API is experimental and might change in + * incompatible ways in the next release. + */ +public abstract class FieldComparator { + + public static final class ByteComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final byte[] values; + private byte[] currentReaderValues; + private final String field; + private ByteParser parser; + + ByteComparator(int numHits, String field, FieldCache.Parser parser) { + values = new byte[numHits]; + this.field = field; + this.parser = (ByteParser) parser; + } + + public int compare(int slot1, int slot2) { + return values[slot1] - values[slot2]; + } + + public int compare(int slot, int doc, float score) { + return (values[slot] - currentReaderValues[doc]); + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? ExtendedFieldCache.EXT_DEFAULT + .getBytes(reader, field, parser) : ExtendedFieldCache.EXT_DEFAULT + .getBytes(reader, field); + } + + public int sortType() { + return SortField.BYTE; + } + + public Comparable value(int slot) { + return new Byte(values[slot]); + } + }; + + public static final class DocComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final int[] docIDs; + private int docBase; + private int readerMaxDoc; + + DocComparator(int numHits) { + docIDs = new int[numHits]; + } + + public int compare(int slot1, int slot2) { + return docIDs[slot1] - docIDs[slot2]; + } + + public int compare(int slot, int doc, float score) { + return docIDs[slot] - docBase - doc; + } + + public void copy(int slot, int doc, float score) { + docIDs[slot] = docBase + doc; + } + + public void setNextReader(IndexReader reader) { + // TODO: can we "map" our docIDs to the current + // reader? saves having to then subtract on every + // compare call + docBase += readerMaxDoc; + readerMaxDoc = reader.maxDoc(); + } + + public int sortType() { + return SortField.DOC; + } + + public Comparable value(int slot) { + return new Integer(docIDs[slot]); + } + }; + + public static final class DoubleComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final double[] values; + private double[] currentReaderValues; + private final String field; + private DoubleParser parser; + + DoubleComparator(int numHits, String field, FieldCache.Parser parser) { + values = new double[numHits]; + this.field = field; + this.parser = (DoubleParser) parser; + } + + public int compare(int slot1, int slot2) { + if (values[slot1] > values[slot2]) + return 1; + if (values[slot1] < values[slot2]) + return -1; + return 0; + } + + public int compare(int slot, int doc, float score) { + if (values[slot] > currentReaderValues[doc]) + return 1; + if (values[slot] < currentReaderValues[doc]) + return -1; + return 0; + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? ExtendedFieldCache.EXT_DEFAULT + .getDoubles(reader, field, parser) : ExtendedFieldCache.EXT_DEFAULT + .getDoubles(reader, field); + } + + public int sortType() { + return SortField.DOUBLE; + } + + public Comparable value(int slot) { + return new Double(values[slot]); + } + }; + + public static final class FloatComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final float[] values; + private float[] currentReaderValues; + private final String field; + private FloatParser parser; + + FloatComparator(int numHits, String field, FieldCache.Parser parser) { + values = new float[numHits]; + this.field = field; + this.parser = (FloatParser) parser; + } + + public int compare(int slot1, int slot2) { + if (values[slot1] > values[slot2]) + return 1; + if (values[slot1] < values[slot2]) + return -1; + return 0; + } + + public int compare(int slot, int doc, float score) { + float val = currentReaderValues[doc]; + if (values[slot] > val) + return 1; + if (values[slot] < val) + return -1; + return 0; + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? FieldCache.DEFAULT.getFloats( + reader, field, parser) : FieldCache.DEFAULT.getFloats(reader, field); + } + + public int sortType() { + return SortField.FLOAT; + } + + public Comparable value(int slot) { + return new Float(values[slot]); + } + }; + + public static final class IntComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final int[] values; + private int[] currentReaderValues; + private final String field; + private IntParser parser; + + IntComparator(int numHits, String field, FieldCache.Parser parser) { + values = new int[numHits]; + this.field = field; + this.parser = (IntParser) parser; + } + + public int compare(int slot1, int slot2) { + if (values[slot1] > values[slot2]) + return 1; + if (values[slot1] < values[slot2]) + return -1; + return 0; + } + + public int compare(int slot, int doc, float score) { + return values[slot] - currentReaderValues[doc]; + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? FieldCache.DEFAULT.getInts(reader, + field, parser) : FieldCache.DEFAULT.getInts(reader, field); + } + + public int sortType() { + return SortField.INT; + } + + public Comparable value(int slot) { + return new Integer(values[slot]); + } + }; + + public static final class LongComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final long[] values; + private long[] currentReaderValues; + private final String field; + private LongParser parser; + + LongComparator(int numHits, String field, FieldCache.Parser parser) { + values = new long[numHits]; + this.field = field; + this.parser = (LongParser) parser; + } + + public int compare(int slot1, int slot2) { + if (values[slot1] > values[slot2]) + return 1; + if (values[slot1] < values[slot2]) + return -1; + return 0; + } + + public int compare(int slot, int doc, float score) { + if (values[slot] > currentReaderValues[doc]) + return 1; + if (values[slot] < currentReaderValues[doc]) + return -1; + return 0; + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? ExtendedFieldCache.EXT_DEFAULT + .getLongs(reader, field, parser) : ExtendedFieldCache.EXT_DEFAULT + .getLongs(reader, field); + } + + public int sortType() { + return SortField.LONG; + } + + public Comparable value(int slot) { + return new Long(values[slot]); + } + }; + + public static final class RelevanceComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final float[] scores; + + RelevanceComparator(int numHits) { + scores = new float[numHits]; + } + + public int compare(int slot1, int slot2) { + float score1 = scores[slot1]; + float score2 = scores[slot2]; + if (score1 > score2) + return -1; + if (score1 < score2) + return 1; + return 0; + } + + public int compare(int slot, int doc, float score) { + float slotScore = scores[slot]; + if (slotScore > score) + return -1; + if (slotScore < score) + return 1; + return 0; + } + + public void copy(int slot, int doc, float score) { + scores[slot] = score; + } + + public void setNextReader(IndexReader reader) { + } + + public int sortType() { + return SortField.SCORE; + } + + public Comparable value(int slot) { + return new Float(scores[slot]); + } + }; + + public static final class ShortComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + + private final short[] values; + private short[] currentReaderValues; + private final String field; + private ShortParser parser; + + ShortComparator(int numHits, String field, FieldCache.Parser parser) { + values = new short[numHits]; + this.field = field; + this.parser = (ShortParser) parser; + } + + public int compare(int slot1, int slot2) { + if (values[slot1] > values[slot2]) + return 1; + if (values[slot1] < values[slot2]) + return -1; + return 0; + } + + public int compare(int slot, int doc, float score) { + if (values[slot] > currentReaderValues[doc]) + return 1; + if (values[slot] < currentReaderValues[doc]) + return -1; + return 0; + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = parser != null ? ExtendedFieldCache.EXT_DEFAULT + .getShorts(reader, field, parser) : ExtendedFieldCache.EXT_DEFAULT + .getShorts(reader, field); + } + + public int sortType() { + return SortField.BYTE; + } + + public Comparable value(int slot) { + return new Short(values[slot]); + } + }; + + public static final class StringComparatorLocale extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + private final String[] values; + private String[] currentReaderValues; + private final String field; + final Collator collator; + + StringComparatorLocale(int numHits, String field, Locale locale) { + values = new String[numHits]; + this.field = field; + collator = Collator.getInstance(locale); + } + + public int compare(int slot1, int slot2) { + final String val1 = values[slot1]; + final String val2 = values[slot2]; + if (val1 == null) { + if (val2 == null) { + return 0; + } + return -1; + } else if (val2 == null) { + return 1; + } + return collator.compare(val1, val2); + } + + public int compare(int slot, int doc, float score) { + return values[slot].compareTo(currentReaderValues[doc]); + } + + public void copy(int slot, int doc, float score) { + values[slot] = currentReaderValues[doc]; + } + + public void setNextReader(IndexReader reader) throws IOException { + currentReaderValues = ExtendedFieldCache.EXT_DEFAULT.getStrings(reader, + field); + } + + public int sortType() { + return SortField.STRING; + } + + public Comparable value(int slot) { + return values[slot]; + } + }; + + public static final class StringOrdComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + private final int[] subords; + private final int[] ords; + private final String[] values; + private String[] lookup; + private int[] order; + private final String field; + + StringOrdComparator(int numHits, String field) { + ords = new int[numHits]; + values = new String[numHits]; + subords = new int[numHits]; + this.field = field; + } + + public int compare(int slot1, int slot2) { + // System.out.println(" x compare slot1=" + slot1 + " val1=" + // + values[slot1] + " slot2=" + slot2 + " val2=" + + // values[slot2]); + final int cmp = ords[slot1] - ords[slot2]; + if (cmp != 0) { + return cmp; + } + + return subords[slot1] - subords[slot2]; + } + + public int compare(int slot, int doc, float score) { + // System.out.println(" e compare slot=" + slot + " val=" + values[slot] + // + " doc=" + doc + " docVal=" + order[doc]); + return ords[slot] - order[doc]; + } + + public void copy(int slot, int doc, float score) { + int ord = order[doc]; + ords[slot] = ord; + values[slot] = lookup[ord]; + // System.out.println(" copy doc=" + doc + " slot=" + slot + " docVal=" + // + lookup[order[doc]]); + } + + public void setNextReader(IndexReader reader) throws IOException { + + // Map ords in queue to ords in reader + + StringIndex currentReaderValues = ExtendedFieldCache.EXT_DEFAULT + .getStringIndex(reader, field); + + lookup = currentReaderValues.lookup; + order = currentReaderValues.order; + int length = lookup.length; + + if (lookup.length == 0) { + return; + } + + // for (int i = 0; i < lookup.length; i++) { + // System.out.println("i:" + i + " " + lookup[i]); + // } + + Map map = new HashMap(); + for (int i = 0; i < values.length; i++) { + String val = values[i]; + if (val == null) { + continue; + } + + int index = binarySearch(lookup, val); + + if (index < 0) { + index = -index - 2; + + Integer intOrd = Integer.valueOf(index); + List slotVals = (List) map.get(intOrd); + if (slotVals == null) { + slotVals = new ArrayList(); + slotVals.add(new SlotValPair(i, val)); + map.put(intOrd, slotVals); + } else { + slotVals.add(new SlotValPair(i, val)); + } + + } else { + subords[i] = 0; + } + ords[i] = index; + } + + Iterator dit = map.values().iterator(); + while (dit.hasNext()) { + List list = (List) dit.next(); + if (list.size() > 1) { + Collections.sort(list); + + for (int i = 0; i < list.size(); i++) { + SlotValPair svp = (SlotValPair) list.get(i); + subords[svp.i] = i + 1; + } + } else { + SlotValPair svp = (SlotValPair) list.get(0); + subords[svp.i] = 1; + } + } + } + + public int sortType() { + return SortField.STRING_ORD; + } + + public Comparable value(int slot) { + return values[slot]; + } + + class SlotValPair implements Comparable { + public SlotValPair(int i, String val) { + this.val = val; + this.i = i; + } + + String val; + int i; + + public int compareTo(Object o) { + SlotValPair odoub = (SlotValPair) o; + return val.compareTo(odoub.val); + } + + public String toString() { + return "i:" + i + " val:" + val; + } + } + } + + public static final class StringOrdValOnDemComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + private final int[] ords; + private final String[] values; + private final int[] readerIndex; + private int currentReaderIndex = -1; + private String[] lookup; + private int length; + private int[] order; + private final String field; + + StringOrdValOnDemComparator(int numHits, String field) { + ords = new int[numHits]; + values = new String[numHits]; + readerIndex = new int[numHits]; + this.field = field; + } + + public int compare(int slot1, int slot2) { + if (readerIndex[slot1] != currentReaderIndex) { + convert(slot1); + } + if (readerIndex[slot2] != currentReaderIndex) { + convert(slot2); + } + int cmp = ords[slot1] - ords[slot2]; + if (cmp != 0) { + return cmp; + } + final String val1 = values[slot1]; + final String val2 = values[slot2]; + if (val1 == null) { + if (val2 == null) { + return 0; + } + return -1; + } else if (val2 == null) { + return 1; + } + + return val1.compareTo(val2); + } + + public int compare(int slot, int doc, float score) { + if (readerIndex[slot] != currentReaderIndex) { + convert(slot); + } + int ord = ords[slot]; + int order = this.order[doc]; + final int cmp = ord - order; + if (cmp != 0) { + return cmp; + } + + final String val1 = values[slot]; + final String val2 = lookup[order]; + if (val1 == null) { + if (val2 == null) { + return 0; + } + return -1; + } else if (val2 == null) { + return 1; + } + return val1.compareTo(val2); + } + + private void convert(int slot) { + readerIndex[slot] = currentReaderIndex; + if (length == 0) { + return; + } + + int index = 0; + String value = values[slot]; + if (value == null) { + ords[slot] = 0; + return; + } + index = binarySearch(lookup, value); + if (index < 0) { + index = -index - 2; + } + ords[slot] = index; + } + + public void copy(int slot, int doc, float score) { + int ord = order[doc]; + ords[slot] = ord; + values[slot] = lookup[ord]; + readerIndex[slot] = currentReaderIndex; + } + + public void setNextReader(IndexReader reader) throws IOException { + StringIndex currentReaderValues = ExtendedFieldCache.EXT_DEFAULT + .getStringIndex(reader, field); + order = currentReaderValues.order; + lookup = currentReaderValues.lookup; + currentReaderIndex++; + length = lookup.length; + } + + public int sortType() { + return SortField.STRING_ORD_VAL; + } + + public Comparable value(int slot) { + return values[slot]; + } + + }; + public static final class StringOrdValOnDemWinComparator extends FieldComparator { + + // nocommit -- maybe "setcurrentscoredoc"? + private final int[] ords; + private final String[] values; + private String[] lookup; + private int length; + private int[] order; + private final String field; + private int WIN = 20; + + StringOrdValOnDemWinComparator(int numHits, String field) { + ords = new int[numHits]; + values = new String[numHits]; + this.field = field; + } + + public int compare(int slot1, int slot2) { + if (ords[slot1] == -1) { + convert(slot1); + } + if (ords[slot2] == -1) { + convert(slot2); + } + int cmp = ords[slot1] - ords[slot2]; + if (cmp != 0) { + return cmp; + } + final String val1 = values[slot1]; + final String val2 = values[slot2]; + if (val1 == null) { + if (val2 == null) { + return 0; + } + return -1; + } else if (val2 == null) { + return 1; + } + return val1.compareTo(val2); + } + + public int compare(int slot, int doc, float score) { + if (ords[slot] == -1) { + convert(slot); + } + int order = this.order[doc]; + int cmp = ords[slot] - order; + + if (cmp != 0) { + return cmp; + } + + final String val1 = values[slot]; + final String val2 = lookup[order]; + if (val1 == null) { + if (val2 == null) { + return 0; + } + return -1; + } else if (val2 == null) { + return 1; + } + return val1.compareTo(val2); + } + + private void convert(int slot) { + if (length == 0) { + return; + } + int start=Math.max(0, slot-WIN); + int end=Math.min(values.length, slot+WIN); + for(int i = start; inull. @@ -206,22 +219,44 @@ /** Creates a sort with a custom comparison function. * @param field Name of field to sort by; cannot be null. * @param comparator Returns a comparator for sorting hits. + * @deprecated use SortField (String field, FieldComparatorSource comparator) */ public SortField (String field, SortComparatorSource comparator) { initFieldType(field, CUSTOM); this.factory = comparator; } + + /** Creates a sort with a custom comparison function. + * @param field Name of field to sort by; cannot be null. + * @param comparator Returns a comparator for sorting hits. + */ + public SortField (String field, FieldComparatorSource comparator) { + initFieldType(field, CUSTOM); + this.comparatorSource = comparator; + } /** Creates a sort, possibly in reverse, with a custom comparison function. * @param field Name of field to sort by; cannot be null. * @param comparator Returns a comparator for sorting hits. * @param reverse True if natural order should be reversed. + * @deprecated use SortField (String field, FieldComparatorSource comparator, boolean reverse) */ public SortField (String field, SortComparatorSource comparator, boolean reverse) { initFieldType(field, CUSTOM); this.reverse = reverse; this.factory = comparator; } + + /** Creates a sort, possibly in reverse, with a custom comparison function. + * @param field Name of field to sort by; cannot be null. + * @param comparator Returns a comparator for sorting hits. + * @param reverse True if natural order should be reversed. + */ + public SortField (String field, FieldComparatorSource comparator, boolean reverse) { + initFieldType(field, CUSTOM); + this.reverse = reverse; + this.comparatorSource = comparator; + } // Sets field & type, and ensures field is not NULL unless // type is SCORE or DOC @@ -273,9 +308,36 @@ return reverse; } + /** + * @deprecated use {@link #getComparatorSource()} + */ public SortComparatorSource getFactory() { return factory; } + + public FieldComparatorSource getComparatorSource() { + return comparatorSource; + } + + /** + * Use legacy IndexSearch semantics: search with a MultiSegmentReader rather + * than passing a single hit collector to multiple SegmentReaders. + * + * @param legacy true for legacy behavior + * @deprecated will be removed in Lucene 3.0. + */ + public void setUseLegacySearch(boolean legacy) { + this.useLegacy = legacy; + } + + /** + * @return if true, IndexSearch will use legacy sorting search semantics. + * eg. multiple Priority Queues. + * @deprecated will be removed in Lucene 3.0. + */ + public boolean getUseLegacySearch() { + return this.useLegacy; + } public String toString() { StringBuffer buffer = new StringBuffer(); Index: src/java/org/apache/lucene/search/BooleanScorer.java =================================================================== --- src/java/org/apache/lucene/search/BooleanScorer.java (revision 728731) +++ src/java/org/apache/lucene/search/BooleanScorer.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.lucene.index.IndexReader; /* Description from Doug Cutting (excerpted from * LUCENE-1483): * @@ -79,11 +80,11 @@ public boolean done; public boolean required = false; public boolean prohibited = false; - public HitCollector collector; + public MultiReaderHitCollector collector; public SubScorer next; public SubScorer(Scorer scorer, boolean required, boolean prohibited, - HitCollector collector, SubScorer next) + MultiReaderHitCollector collector, SubScorer next) throws IOException { this.scorer = scorer; this.done = !scorer.next(); @@ -248,12 +249,12 @@ public final int size() { return SIZE; } - public HitCollector newCollector(int mask) { + public MultiReaderHitCollector newCollector(int mask) { return new Collector(mask, this); } } - static final class Collector extends HitCollector { + static final class Collector extends MultiReaderHitCollector { private BucketTable bucketTable; private int mask; public Collector(int mask, BucketTable bucketTable) { @@ -281,6 +282,9 @@ bucket.coord++; // increment coord } } + public void setNextReader(IndexReader reader) { + // not needed by this implementation + } } public boolean skipTo(int target) { Index: src/java/org/apache/lucene/search/MultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/MultiSearcher.java (revision 728731) +++ src/java/org/apache/lucene/search/MultiSearcher.java (working copy) @@ -205,7 +205,7 @@ totalHits += docs.totalHits; // update totalHits ScoreDoc[] scoreDocs = docs.scoreDocs; for (int j = 0; j < scoreDocs.length; j++) { // merge scoreDocs into hq - ScoreDoc scoreDoc = scoreDocs[j]; + ScoreDoc scoreDoc = scoreDocs[j]; scoreDoc.doc += starts[i]; // convert doc if(!hq.insert(scoreDoc)) break; // no more scores > minScore @@ -255,15 +255,7 @@ public void search(Weight weight, Filter filter, final HitCollector results) throws IOException { for (int i = 0; i < searchables.length; i++) { - - final int start = starts[i]; - - searchables[i].search(weight, filter, new HitCollector() { - public void collect(int doc, float score) { - results.collect(doc + start, score); - } - }); - + searchables[i].search(weight, filter, results); } } Index: src/java/org/apache/lucene/search/QueryWrapperFilter.java =================================================================== --- src/java/org/apache/lucene/search/QueryWrapperFilter.java (revision 728731) +++ src/java/org/apache/lucene/search/QueryWrapperFilter.java (working copy) @@ -51,10 +51,16 @@ public BitSet bits(IndexReader reader) throws IOException { final BitSet bits = new BitSet(reader.maxDoc()); - new IndexSearcher(reader).search(query, new HitCollector() { + new IndexSearcher(reader).search(query, new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public final void collect(int doc, float score) { - bits.set(doc); // set bit for hit + bits.set(doc + base); // set bit for hit } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); return bits; } Index: src/java/org/apache/lucene/search/TimeLimitedCollector.java =================================================================== --- src/java/org/apache/lucene/search/TimeLimitedCollector.java (revision 728731) +++ src/java/org/apache/lucene/search/TimeLimitedCollector.java (working copy) @@ -1,5 +1,7 @@ package org.apache.lucene.search; +import org.apache.lucene.index.IndexReader; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -22,8 +24,9 @@ * take longer than the maximum allowed search time limit. After this * time is exceeded, the search thread is stopped by throwing a * TimeExceeded Exception.

+ * */ -public class TimeLimitedCollector extends HitCollector { +public class TimeLimitedCollector extends MultiReaderHitCollector { /** * Default timer resolution. @@ -140,6 +143,10 @@ private final long timeout; private final HitCollector hc; + private int docBase = -1; + + private int lastMax; + /** * Create a TimeLimitedCollector wrapper over another HitCollector with a specified timeout. * @param hc the wrapped HitCollector @@ -161,13 +168,13 @@ if( timeout < time) { if (greedy) { //System.out.println(this+" greedy: before failing, collecting doc: "+doc+" "+(time-t0)); - hc.collect( doc, score ); + hc.collect( doc + docBase, score ); } //System.out.println(this+" failing on: "+doc+" "+(time-t0)); throw new TimeExceededException( timeout-t0, time-t0, doc ); } //System.out.println(this+" collecting: "+doc+" "+(time-t0)); - hc.collect( doc, score ); + hc.collect( doc + docBase, score ); } /** @@ -216,4 +223,9 @@ public void setGreedy(boolean greedy) { this.greedy = greedy; } + + public void setNextReader(IndexReader reader) { + docBase = (docBase == -1) ? 0 : docBase + lastMax; + lastMax = reader.maxDoc(); + } } Index: src/java/org/apache/lucene/search/TopFieldCollector.java =================================================================== --- src/java/org/apache/lucene/search/TopFieldCollector.java (revision 0) +++ src/java/org/apache/lucene/search/TopFieldCollector.java (revision 0) @@ -0,0 +1,162 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; + +/** + * A {@link HitCollector} that sorts by {@link SortField} using + * {@link FieldComparator}s. + * + * NOTE: This API is experimental and might change in + * incompatible ways in the next release. + */ +public class TopFieldCollector extends MultiReaderHitCollector { + + private final FieldValueHitQueue queue; + + private final FieldComparator[] comparators; + private final FieldComparator comparator1; + private final int numComparators; + private final int[] reverseMul; + private final int reverseMul1; + + private int subReaderIndex = -1; + + private final int numHits; + private int totalHits; + private FieldValueHitQueue.Entry weakest = null; + + /** Stores the maximum score value encountered, needed for normalizing. */ + private float maxScore = Float.NEGATIVE_INFINITY; + + private boolean queueFull; + + public TopFieldCollector(Sort sort, int numHits) + throws IOException { + queue = new FieldValueHitQueue(sort.fields, numHits); + comparators = queue.getComparators(); + reverseMul = queue.getReverseMul(); + numComparators = comparators.length; + if (numComparators > 0) { + comparator1 = comparators[0]; + reverseMul1 = reverseMul[0]; + } else { + comparator1 = null; + reverseMul1 = 0; + } + + this.numHits = numHits; + } + + public void setNextReader(IndexReader reader) throws IOException { + subReaderIndex++; + for (int i = 0; i < numComparators; i++) + comparators[i].setNextReader(reader); + } + + // javadoc inherited + public void collect(int doc, float score) { + + if (score > 0.0f) { + + maxScore = Math.max(maxScore, score); + totalHits++; + + if (queueFull) { + + // Fastmatch: return if this hit is not competitive + if (numComparators == 1) { + // Common case + if (reverseMul1 * comparator1.compare(weakest.slot, doc, score) <= 0) { + // Definitely not competitive + return; + } + + // This hit is competitive -- replace weakest + // element in queue & adjustTop + comparator1.copy(weakest.slot, doc, score); + } else { + for(int i=0;;i++) { + final int c = reverseMul[i] * comparators[i].compare(weakest.slot, doc, score); + if (c < 0) { + // Definitely not competitive + return; + } else if (c > 0) { + // Definitely competitive + break; + } else if (i == numComparators-1) { + // This is the equals case. We don't need to + // fallback to docID comparison because lower + // docIDs always win, and this docID must be + // greater than all docIDs in the queue now, + // so this hit is not competitive. + return; + } + } + + // This hit is competitive -- replace weakest + // element in queue & adjustTop + for (int i = 0; i < numComparators; i++) { + comparators[i].copy(weakest.slot, doc, score); + } + } + + weakest.subReaderIndex = subReaderIndex; + weakest.subDocID = doc; + weakest.score = score; + queue.adjustTop(); + weakest = (FieldValueHitQueue.Entry) queue.top(); + } else { + // Startup transient: queue hasn't gathered numHits + // yet + + final int slot = totalHits-1; + + // Copy hit into queue + if (numComparators == 1) { + // Common case + comparator1.copy(slot, doc, score); + } else { + for (int i = 0; i < numComparators; i++) { + comparators[i].copy(slot, doc, score); + } + } + + queue.put(new FieldValueHitQueue.Entry(slot, + subReaderIndex, doc, score)); + + weakest = (FieldValueHitQueue.Entry) queue.top(); + queueFull = totalHits == numHits; + } + } + } + + // javadoc inherited + public TopDocs topDocs(int[] starts) { + ScoreDoc[] scoreDocs = new ScoreDoc[queue.size()]; + for (int i = queue.size() - 1; i >= 0; i--) { + scoreDocs[i] = queue.fillFields((FieldValueHitQueue.Entry) queue + .pop(), starts); + } + + return new TopFieldDocs(totalHits, scoreDocs, queue.getFields(), maxScore); + } +} Property changes on: src/java/org/apache/lucene/search/TopFieldCollector.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/search/ParallelMultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/ParallelMultiSearcher.java (revision 728731) +++ src/java/org/apache/lucene/search/ParallelMultiSearcher.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.util.PriorityQueue; @@ -170,10 +171,16 @@ final int start = starts[i]; - searchables[i].search(weight, filter, new HitCollector() { + searchables[i].search(weight, filter, new MultiReaderHitCollector() { + private int base = -1; + private int lastMax; public void collect(int doc, float score) { - results.collect(doc + start, score); + results.collect(doc + base + start, score); } + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } }); } Index: src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- src/java/org/apache/lucene/search/IndexSearcher.java (revision 728731) +++ src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -17,6 +17,8 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.CorruptIndexException; @@ -24,8 +26,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; -import java.io.IOException; - /** Implements search over a single IndexReader. * *

Applications usually need only call the inherited {@link #search(Query)} @@ -38,6 +38,7 @@ public class IndexSearcher extends Searcher { IndexReader reader; private boolean closeReader; + private final int[] starts; /** Creates a searcher searching the index in the named directory. * @throws CorruptIndexException if the index is corrupt @@ -63,6 +64,13 @@ private IndexSearcher(IndexReader r, boolean closeReader) { reader = r; this.closeReader = closeReader; + IndexReader[] subReaders = r.getSequentialReaders(); + starts = new int[subReaders.length]; + int maxDoc = 0; + for (int i = 0; i < subReaders.length; i++) { + starts[i] = maxDoc; + maxDoc += subReaders[i].maxDoc(); // compute maxDocs + } } /** Return the {@link IndexReader} this searches. */ @@ -104,30 +112,95 @@ // inherit javadoc public TopDocs search(Weight weight, Filter filter, final int nDocs) throws IOException { - - if (nDocs <= 0) // null might be returned from hq.top() below. - throw new IllegalArgumentException("nDocs must be > 0"); - TopDocCollector collector = new TopDocCollector(nDocs); - search(weight, filter, collector); - return collector.topDocs(); + IndexReader[] readers = reader.getSequentialReaders(); + if (readers.length > 1) { + for (int i = 0; i < readers.length; i++) { // search each searcher + collector.setNextReader(readers[i]); + doSearch(readers[i], weight, filter, collector); + } + } else { + collector.setNextReader(reader); + doSearch(reader, weight, filter, collector); + } + return (TopDocs) collector.topDocs(); } // inherit javadoc public TopFieldDocs search(Weight weight, Filter filter, final int nDocs, Sort sort) throws IOException { + + SortField[] fields = sort.fields; + boolean legacy = false; + for(int i = 0; i < fields.length; i++) { + SortField field = fields[i]; + String fieldname = field.getField(); + int type = field.getType(); + if (type == SortField.AUTO) { + int autotype = FieldValueHitQueue.detectFieldType(reader, fieldname); + if (autotype == SortField.STRING) { + fields[i] = new SortField (fieldname, field.getLocale(), field.getReverse()); + } else { + fields[i] = new SortField (fieldname, autotype, field.getReverse()); + } + } else if(type == SortField.CUSTOM) { // Remove in Lucene 3.0 + if(field.getFactory() != null) { + legacy = true; + } + } + if(field.getUseLegacySearch()) { + legacy = true; + } + } + + if(legacy) { // use multiple queues + + TopDocCollector collector = new TopFieldDocCollector(reader, sort, nDocs); - TopFieldDocCollector collector = - new TopFieldDocCollector(reader, sort, nDocs); + collector.setNextReader(reader); + + doSearch(reader, weight, filter, collector); + + return (TopFieldDocs) collector.topDocs(); + } + // use a single queue + TopFieldCollector collector = new TopFieldCollector(sort, nDocs); search(weight, filter, collector); - return (TopFieldDocs)collector.topDocs(); + + return (TopFieldDocs) collector.topDocs(starts); } + // inherit javadoc - public void search(Weight weight, Filter filter, - final HitCollector results) throws IOException { + public void search(Weight weight, Filter filter, HitCollector results) + throws IOException { + IndexReader[] readers = reader.getSequentialReaders(); + HitCollector collector = results; + if (readers.length > 1) { + if(!(results instanceof MultiReaderHitCollector)) { + collector = new MultiReaderCollectorWrapper(results); + } + for (int i = 0; i < readers.length; i++) { // search each searcher + ((MultiReaderHitCollector) collector).setNextReader(readers[i]); + doSearch(readers[i], weight, filter, collector); + } + + } else { + if(!(results instanceof MultiReaderHitCollector)) { + collector = new MultiReaderCollectorWrapper(results); + } + + ((MultiReaderHitCollector) collector).setNextReader(reader); + + doSearch(reader, weight, filter, collector); + } + } + + private void doSearch(IndexReader reader, Weight weight, Filter filter, + final HitCollector results) throws IOException { + Scorer scorer = weight.scorer(reader); if (scorer == null) return; @@ -138,7 +211,7 @@ } DocIdSetIterator filterDocIdIterator = filter.getDocIdSet(reader).iterator(); // CHECKME: use ConjunctionScorer here? - + boolean more = filterDocIdIterator.next() && scorer.skipTo(filterDocIdIterator.doc()); while (more) { @@ -156,7 +229,7 @@ } } } - + public Query rewrite(Query original) throws IOException { Query query = original; for (Query rewrittenQuery = query.rewrite(reader); rewrittenQuery != query; @@ -169,4 +242,26 @@ public Explanation explain(Weight weight, int doc) throws IOException { return weight.explain(reader, doc); } + + /** + * Wrapper for non expert HitCollector implementations. + */ + private class MultiReaderCollectorWrapper extends MultiReaderHitCollector { + private HitCollector collector; + private int base = -1; + private int lastMax; + + public MultiReaderCollectorWrapper(HitCollector collector) { + this.collector = collector; + } + + public void collect(int doc, float score) { + collector.collect(doc + base, score); + } + + public void setNextReader(IndexReader reader) { + base = (base == -1) ? 0 : base + lastMax; + lastMax = reader.maxDoc(); + } + } } Index: src/java/org/apache/lucene/search/FieldComparatorSource.java =================================================================== --- src/java/org/apache/lucene/search/FieldComparatorSource.java (revision 0) +++ src/java/org/apache/lucene/search/FieldComparatorSource.java (revision 0) @@ -0,0 +1,43 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Serializable; + +/** + * nocommit -- needs javadoc + * + * NOTE: This API is experimental and might change in + * incompatible ways in the next release. + * + */ +public interface FieldComparatorSource extends Serializable { + + /** + * Creates a comparator for the field in the given index. + * + * @param fieldname + * Name of the field to create comparator for. + * @return FieldComparator. + * @throws IOException + * If an error occurs reading the index. + */ + FieldComparator newComparator(String fieldname) + throws IOException; +} Property changes on: src/java/org/apache/lucene/search/FieldComparatorSource.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/index/MultiReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiReader.java (revision 728731) +++ src/java/org/apache/lucene/index/MultiReader.java (working copy) @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -365,7 +364,11 @@ throw new UnsupportedOperationException("MultiReader does not support this method."); } - // for testing + public IndexReader[] getSequentialReaders() { + return subReaders; + } + + // for testing IndexReader[] getSubReaders() { return subReaders; } Index: src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/FilterIndexReader.java (revision 728731) +++ src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -227,4 +227,8 @@ ensureOpen(); return in.isOptimized(); } + + public IndexReader[] getSequentialReaders() { + return in.getSequentialReaders(); + } } Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 728731) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1134,4 +1134,9 @@ public static Collection listCommits(Directory dir) throws IOException { return DirectoryIndexReader.listCommits(dir); } + + public IndexReader[] getSequentialReaders() { + return new IndexReader[] {this}; + } + } Index: src/java/org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 728731) +++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -19,7 +19,6 @@ import java.io.IOException; import java.util.Collection; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -432,6 +431,10 @@ return fieldSet; } + public IndexReader[] getSequentialReaders() { + return subReaders; + } + // for testing SegmentReader[] getSubReaders() { return subReaders; Index: src/java/org/apache/lucene/util/PriorityQueue.java =================================================================== --- src/java/org/apache/lucene/util/PriorityQueue.java (revision 728731) +++ src/java/org/apache/lucene/util/PriorityQueue.java (working copy) @@ -163,4 +163,12 @@ } heap[i] = node; // install saved node } + + //TODO: remove + public void print() { + System.out.println("queue"); + for(int i = 0; i < size; i++) { + System.out.println(heap[i]); + } + } } Index: contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java =================================================================== --- contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java (revision 728731) +++ contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java (working copy) @@ -16,9 +16,9 @@ package org.apache.lucene.store.instantiated; import junit.framework.TestCase; + +import org.apache.lucene.search.HitCollector; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TopDocCollector; -import org.apache.lucene.search.HitCollector; import org.apache.lucene.search.TermQuery; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithSortTask.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithSortTask.java (revision 728731) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/SearchWithSortTask.java (working copy) @@ -45,18 +45,24 @@ SortField[] sortFields = new SortField[fields.length]; for (int i = 0; i < fields.length; i++) { String field = fields[i]; - int index = field.lastIndexOf(":"); - String fieldName; - String typeString; - if (index != -1) { - fieldName = field.substring(0, index); - typeString = field.substring(1+index, field.length()); + SortField sortField0; + if (field.equals("doc")) { + sortField0 = SortField.FIELD_DOC; } else { - typeString = "auto"; - fieldName = field; + int index = field.lastIndexOf(":"); + String fieldName; + String typeString; + if (index != -1) { + fieldName = field.substring(0, index); + typeString = field.substring(1+index, field.length()); + } else { + typeString = "auto"; + fieldName = field; + } + int type = getType(typeString); + sortField0 = new SortField(fieldName, type); } - int type = getType(typeString); - sortFields[i] = new SortField(fieldName, type); + sortFields[i] = sortField0; } this.sort = new Sort(sortFields); } @@ -69,6 +75,18 @@ type = SortField.INT; } else if (typeString.equals("string")) { type = SortField.STRING; + } else if (typeString.equals("string_ord")) { + type = SortField.STRING_ORD; + } else if (typeString.equals("string_ord_val")) { + type = SortField.STRING_ORD_VAL; + } else if (typeString.equals("string_ord_val_dem")) { + type = SortField.STRING_ORD_VAL_DEM; + //} else if (typeString.equals("string_ord_val_lazy")) { + //type = SortField.STRING_ORD_VAL_LAZY; + //} else if (typeString.equals("string_ord_val2")) { + //type = SortField.STRING_ORD_VAL2; + } else if (typeString.equals("string_val")) { + type = SortField.STRING_VAL; } else { type = SortField.AUTO; } Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (revision 728731) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/ReadTask.java (working copy) @@ -62,6 +62,9 @@ super(runData); } + // nocommit + static boolean first = true; + public int doLogic() throws Exception { int res = 0; boolean closeReader = false; @@ -100,8 +103,13 @@ } else { hits = searcher.search(q, numHits); } - //System.out.println("q=" + q + ":" + hits.totalHits + " total hits"); - + //System.out.println("q=" + q + ":" + hits.totalHits + //+ " total hits"); + // nocommit + if (first) { + System.out.println("NUMHITS=" + hits.totalHits); + first = false; + } if (withTraverse()) { final ScoreDoc[] scoreDocs = hits.scoreDocs; int traversalSize = Math.min(scoreDocs.length, traversalSize()); Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SortableSimpleDocMaker.java =================================================================== --- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SortableSimpleDocMaker.java (revision 728731) +++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/SortableSimpleDocMaker.java (working copy) @@ -11,15 +11,38 @@ */ public class SortableSimpleDocMaker extends SimpleDocMaker { private int sortRange; + + private static String[] COUNTRIES = new String[] {"European Union", "United States", "Japan", "Germany", "China (PRC)", "United Kingdom", "France", "Italy", "Spain", "Canada", "Brazil", "Russia", "India", "South Korea", "Australia", "Mexico", "Netherlands", "Turkey", "Sweden", "Belgium", "Indonesia", "Switzerland", "Poland", "Norway", "Republic of China", "Saudi Arabia", "Austria", "Greece", "Denmark", "Iran", "South Africa", "Argentina", "Ireland", "Thailand", "Finland", "Venezuela", "Portugal", "Hong Kong", "United Arab Emirates", "Malaysia", "Czech Republic", "Colombia", "Nigeria", "Romania", "Chile", "Israel", "Singapore", "Philippines", "Pakistan", "Ukraine", "Hungary", "Algeria", "New Zealand", "Egypt", "Kuwait", "Peru", "Kazakhstan", "Slovakia", "Morocco", "Bangladesh", "Vietnam", "Qatar", "Angola", "Libya", "Iraq", "Croatia", "Luxembourg", "Sudan", "Slovenia", "Cuba", "Belarus", "Ecuador", "Serbia", "Oman", "Bulgaria", "Lithuania", "Syria", "Dominican Republic", "Tunisia", "Guatemala", "Azerbaijan", "Sri Lanka", "Kenya", "Latvia", "Turkmenistan", "Costa Rica", "Lebanon", "Uruguay", "Uzbekistan", "Yemen", "Cyprus", "Estonia", "Trinidad and Tobago", "Cameroon", "El Salvador", "Iceland", "Panama", "Bahrain", "Ivory Coast", "Ethiopia", "Tanzania", "Jordan", "Ghana", "Bosnia and Herzegovina", "Macau", "Burma", "Bolivia", "Brunei", "Botswana", "Honduras", "Gabon", "Uganda", "Jamaica", "Zambia", "Senegal", "Paraguay", "Albania", "Equatorial Guinea", "Georgia", "Democratic Republic of the Congo", "Nepal", "Afghanistan", "Cambodia", "Armenia", "Republic of the Congo", "Mozambique", "Republic of Macedonia", "Malta", "Namibia", "Madagascar", "Chad", "Burkina Faso", "Mauritius", "Mali", "The Bahamas", "Papua New Guinea", "Nicaragua", "Haiti", "Benin", "alestinian flag West Bank and Gaza", "Jersey", "Fiji", "Guinea", "Moldova", "Niger", "Laos", "Mongolia", "French Polynesia", "Kyrgyzstan", "Barbados", "Tajikistan", "Malawi", "Liechtenstein", "New Caledonia", "Kosovo", "Rwanda", "Montenegro", "Swaziland", "Guam", "Mauritania", "Guernsey", "Isle of Man", "Togo", "Somalia", "Suriname", "Aruba", "North Korea", "Zimbabwe", "Central African Republic", "Faroe Islands", "Greenland", "Sierra Leone", "Lesotho", "Cape Verde", "Eritrea", "Bhutan", "Belize", "Antigua and Barbuda", "Gibraltar", "Maldives", "San Marino", "Guyana", "Burundi", "Saint Lucia", "Djibouti", "British Virgin Islands", "Liberia", "Seychelles", "The Gambia", "Northern Mariana Islands", "Grenada", "Saint Vincent and the Grenadines", "Saint Kitts and Nevis", "East Timor", "Vanuatu", "Comoros", "Samoa", "Solomon Islands", "Guinea-Bissau", "American Samoa", "Dominica", "Micronesia", "Tonga", "Cook Islands", "Palau", "Marshall Islands", "São Tomé and Príncipe", "Anguilla", "Kiribati", "Tuvalu", "Niue"}; protected DocData getNextDocData() throws NoMoreDataException { + Random r = new Random(); DocData doc = super.getNextDocData(); Properties props = new Properties(); - props.put("sort_field", Integer.toString(getRandomNumber(0, sortRange))); + + // random int + props.put("sort_field", Integer.toString(nextInt(r, sortRange))); + + // random string + int len = nextInt(r, 2, 20); + char[] buffer = new char[len]; + for(int i=0;i