Index: lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java (révision 1461822) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/TestSortingMergePolicy.java (copie de travail) @@ -68,7 +68,7 @@ return doc; } - private static MergePolicy newSortingMergePolicy(Sorter sorter) { + static MergePolicy newSortingMergePolicy(Sorter sorter) { // create a MP with a low merge factor so that many merges happen MergePolicy mp; if (random().nextBoolean()) { Index: lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java (révision 0) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java (copie de travail) @@ -0,0 +1,143 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +public class TestEarlyTermination extends LuceneTestCase { + + int numDocs; + private List terms; + private Directory dir; + private Sorter sorter; + private RandomIndexWriter iw; + private IndexReader reader; + + @Override + public void setUp() throws Exception { + super.setUp(); + sorter = new NumericDocValuesSorter("ndv"); + } + + private Document randomDocument() { + final Document doc = new Document(); + doc.add(new NumericDocValuesField("ndv", random().nextInt(10))); + doc.add(new StringField("s", RandomPicks.randomFrom(random(), terms), Store.YES)); + return doc; + } + + private void createRandomIndexes(int maxSegments) throws IOException { + dir = newDirectory(); + numDocs = atLeast(150); + final int numTerms = _TestUtil.nextInt(random(), 1, numDocs / 5); + Set randomTerms = new HashSet(); + while (randomTerms.size() < numTerms) { + randomTerms.add(_TestUtil.randomSimpleString(random())); + } + terms = new ArrayList(randomTerms); + final long seed = random().nextLong(); + final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed))); + iwc.setMergePolicy(TestSortingMergePolicy.newSortingMergePolicy(sorter)); + iw = new RandomIndexWriter(new Random(seed), dir, iwc); + for (int i = 0; i < numDocs; ++i) { + final Document doc = randomDocument(); + iw.addDocument(doc); + if (i == numDocs / 2 || (i != numDocs - 1 && random().nextInt(8) == 0)) { + iw.commit(); + } + if (random().nextInt(15) == 0) { + final String term = RandomPicks.randomFrom(random(), terms); + iw.deleteDocuments(new Term("s", term)); + } + } + reader = iw.getReader(); + } + + @Override + public void tearDown() throws Exception { + reader.close(); + iw.close(); + dir.close(); + super.tearDown(); + } + + public void testEarlyTermination() throws IOException { + createRandomIndexes(5); + final int numHits = _TestUtil.nextInt(random(), 1, numDocs / 10); + final Sort sort = new Sort(new SortField("ndv", SortField.Type.LONG, false)); + final boolean fillFields = random().nextBoolean(); + final boolean trackDocScores = random().nextBoolean(); + final boolean trackMaxScore = random().nextBoolean(); + final boolean inOrder = random().nextBoolean(); + final TopFieldCollector collector1 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder); + final TopFieldCollector collector2 = TopFieldCollector.create(sort, numHits, fillFields, trackDocScores, trackMaxScore, inOrder); + + final IndexSearcher searcher = newSearcher(reader); + final int iters = atLeast(5); + for (int i = 0; i < iters; ++i) { + final TermQuery query = new TermQuery(new Term("s", RandomPicks.randomFrom(random(), terms))); + searcher.search(query, collector1); + searcher.search(query, new SortingMergePolicyCollector(collector2) { + @Override + protected int numDocsToCollect(AtomicReaderContext context) { + return numHits; + } + }); + } + assertTrue(collector1.getTotalHits() >= collector2.getTotalHits()); + assertTopDocsEquals(collector1.topDocs().scoreDocs, collector2.topDocs().scoreDocs); + } + + private static void assertTopDocsEquals(ScoreDoc[] scoreDocs1, ScoreDoc[] scoreDocs2) { + assertEquals(scoreDocs1.length, scoreDocs2.length); + for (int i = 0; i < scoreDocs1.length; ++i) { + final ScoreDoc scoreDoc1 = scoreDocs1[i]; + final ScoreDoc scoreDoc2 = scoreDocs2[i]; + assertEquals(scoreDoc1.doc, scoreDoc2.doc); + assertEquals(scoreDoc1.score, scoreDoc2.score, 0.001f); + } + } + +} Modification de propriétés sur lucene/misc/src/test/org/apache/lucene/index/sorter/TestEarlyTermination.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicyCollector.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicyCollector.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicyCollector.java (copie de travail) @@ -0,0 +1,52 @@ + package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.util.Map; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.search.Collector; + +/** A {@link Collector} that assumes that merged segments are sorted and + * terminates collection early. This {@link Collector} is best used in + * conjunction with {@link SortingMergePolicy} when the sort order of the + * query matches the sort order of the index. + * @lucene.experimental */ +public abstract class SortingMergePolicyCollector extends EarlyTerminationCollector { + + /** Sole constructor, typically used for inheritance. */ + protected SortingMergePolicyCollector(Collector in) { + super(in); + } + + @Override + protected boolean sorted(AtomicReader reader) { + if (reader instanceof SegmentReader) { + final SegmentReader segReader = (SegmentReader) reader; + final Map diagnostics = segReader.getSegmentInfo().info.getDiagnostics(); + if (diagnostics != null && IndexWriter.SOURCE_MERGE.equals(diagnostics.get(IndexWriter.SOURCE))) { + return true; + } + } + return false; + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/SortingMergePolicyCollector.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminationCollector.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminationCollector.java (révision 0) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminationCollector.java (copie de travail) @@ -0,0 +1,88 @@ +package org.apache.lucene.index.sorter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.Scorer; + +/** A {@link Collector} that early-terminates collection on sorted segments. + * @lucene.experimental */ +public abstract class EarlyTerminationCollector extends Collector { + + final Collector in; + int totalCollect; + int numCollected; + boolean sorted; + + /** Sole constructor, typically used for inheritance. */ + protected EarlyTerminationCollector(Collector in) { + this.in = in; + } + + /** Returns whether the provided reader is sorted. This method is called for + * every reader leaf on every search and should therefore be fast. In + * particular, it shouldn't inspect every document of the reader to check + * whether it is sorted. */ + protected abstract boolean sorted(AtomicReader reader); + + /** Returns the maximum number of documents that should be collected on the + * current sorted leaf. */ + protected abstract int numDocsToCollect(AtomicReaderContext context); + + @Override + public void setScorer(Scorer scorer) throws IOException { + in.setScorer(scorer); + } + + @Override + public void collect(int doc) throws IOException { + in.collect(doc); + if (++numCollected >= totalCollect) { + throw new CollectionTerminatedException(); + } + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + in.setNextReader(context); + sorted = sorted(context.reader()); + if (sorted) { + totalCollect = numDocsToCollect(context); + if (totalCollect < 0) { + throw new IllegalStateException("numDocsToCollect must always return a result >= 0, got " + totalCollect); + } + if (totalCollect == 0) { + throw new CollectionTerminatedException(); + } + } else { + totalCollect = Integer.MAX_VALUE; + } + numCollected = 0; + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return !sorted && in.acceptsDocsOutOfOrder(); + } + +} Modification de propriétés sur lucene/misc/src/java/org/apache/lucene/index/sorter/EarlyTerminationCollector.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java (révision 1461822) +++ lucene/core/src/test/org/apache/lucene/search/TestEarlyTermination.java (copie de travail) @@ -77,7 +77,12 @@ @Override public void setNextReader(AtomicReaderContext context) throws IOException { - collectionTerminated = false; + if (random().nextBoolean()) { + collectionTerminated = true; + throw new CollectionTerminatedException(); + } else { + collectionTerminated = false; + } } @Override Index: lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (révision 1461822) +++ lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (copie de travail) @@ -591,7 +591,13 @@ // threaded...? the Collector could be sync'd? // always use single thread: for (AtomicReaderContext ctx : leaves) { // search each subreader - collector.setNextReader(ctx); + try { + collector.setNextReader(ctx); + } catch (CollectionTerminatedException e) { + // there is no doc of interest in this reader context + // continue with the following leaf + continue; + } Scorer scorer = weight.scorer(ctx, !collector.acceptsDocsOutOfOrder(), true, ctx.reader().getLiveDocs()); if (scorer != null) { try { Index: lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (révision 1461822) +++ lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (copie de travail) @@ -555,7 +555,7 @@ SegmentInfoPerCommit newSegment = flushedSegment.segmentInfo; - IndexWriter.setDiagnostics(newSegment.info, "flush"); + IndexWriter.setDiagnostics(newSegment.info, IndexWriter.SOURCE_FLUSH); IOContext context = new IOContext(new FlushInfo(newSegment.info.getDocCount(), newSegment.sizeInBytes())); Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (révision 1461822) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (copie de travail) @@ -185,13 +185,21 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { private static final int UNBOUNDED_MAX_MERGE_SEGMENTS = -1; - /** * Name of the write lock in the index. */ public static final String WRITE_LOCK_NAME = "write.lock"; + /** Key for the source of a segment in the {@link SegmentInfo#getDiagnostics() diagnostics}. */ + public static final String SOURCE = "source"; + /** Source of a segment which results from a merge of other segments. */ + public static final String SOURCE_MERGE = "merge"; + /** Source of a segment which results from a flush. */ + public static final String SOURCE_FLUSH = "flush"; + /** Source of a segment which results from a call to {@link #addIndexes(IndexReader...)}. */ + public static final String SOURCE_ADDINDEXES_READERS = "addIndexes(IndexReader...)"; + /** * Absolute hard maximum length for a term, in bytes once * encoded as UTF8. If a term arrives from the analyzer @@ -2432,7 +2440,7 @@ info.setFiles(new HashSet(trackingDir.getCreatedFiles())); trackingDir.getCreatedFiles().clear(); - setDiagnostics(info, "addIndexes(IndexReader...)"); + setDiagnostics(info, SOURCE_ADDINDEXES_READERS); boolean useCompoundFile; synchronized(this) { // Guard segmentInfos @@ -3463,7 +3471,7 @@ Map details = new HashMap(); details.put("mergeMaxNumSegments", ""+merge.maxNumSegments); details.put("mergeFactor", Integer.toString(merge.segments.size())); - setDiagnostics(si, "merge", details); + setDiagnostics(si, SOURCE_MERGE, details); if (infoStream.isEnabled("IW")) { infoStream.message("IW", "merge seg=" + merge.info.info.name + " " + segString(merge.segments)); Index: lucene/core/src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (révision 1461822) +++ lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (copie de travail) @@ -198,7 +198,7 @@ /** * Return the SegmentInfoPerCommit of the segment this reader is reading. */ - SegmentInfoPerCommit getSegmentInfo() { + public SegmentInfoPerCommit getSegmentInfo() { return si; }