Index: lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (révision 1343109) +++ lucene/core/src/test/org/apache/lucene/index/TestSegmentMerger.java (copie de travail) @@ -24,9 +24,11 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.InfoStream; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.packed.PackedInts; public class TestSegmentMerger extends LuceneTestCase { @@ -134,4 +136,41 @@ TestSegmentReader.checkNorms(mergedReader); mergedReader.close(); } + + private static boolean equals(MergeState.DocMap map1, MergeState.DocMap map2) { + if (map1.maxDoc() != map2.maxDoc()) { + return false; + } + for (int i = 0; i < map1.maxDoc(); ++i) { + if (map1.get(i) != map2.get(i)) { + return false; + } + } + return true; + } + + public void testBuildDocMap() { + final int maxDoc = 128; + final FixedBitSet liveDocs = new FixedBitSet(maxDoc); + + MergeState.DocMap docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, maxDoc, liveDocs, PackedInts.COMPACT); + MergeState.DocMap docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, 0, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + + liveDocs.set(1); + for (int i = 7; i < 79; ++i) { + liveDocs.set(i); + } + liveDocs.set(80); + liveDocs.set(88); + int numDocs = liveDocs.cardinality(); + docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, maxDoc - numDocs, liveDocs, PackedInts.COMPACT); + docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, numDocs, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + + liveDocs.set(0, maxDoc); + docMap1 = MergeState.DocMap.buildDelCountDocmap(maxDoc, 0, liveDocs, PackedInts.COMPACT); + docMap2 = MergeState.DocMap.buildDirectDocMap(maxDoc, maxDoc, liveDocs, PackedInts.COMPACT); + assertTrue(equals(docMap1, docMap2)); + } } Index: lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java (révision 1343109) +++ lucene/core/src/java/org/apache/lucene/index/SortedBytesMergeUtils.java (copie de travail) @@ -81,7 +81,8 @@ } } - public static List buildSlices(int[] docBases, int[][] docMaps, + public static List buildSlices( + int[] docBases, MergeState.DocMap[] docMaps, DocValues[] docValues, MergeContext ctx) throws IOException { final List slices = new ArrayList(); for (int i = 0; i < docValues.length; i++) { @@ -111,15 +112,15 @@ * mapping in docIDToRelativeOrd. After the merge SortedSourceSlice#ordMapping * contains the new global ordinals for the relative index. */ - private static void createOrdMapping(int[] docBases, int[][] docMaps, + private static void createOrdMapping(int[] docBases, MergeState.DocMap[] docMaps, SortedSourceSlice currentSlice) { final int readerIdx = currentSlice.readerIdx; - final int[] currentDocMap = docMaps[readerIdx]; + final MergeState.DocMap currentDocMap = docMaps[readerIdx]; final int docBase = currentSlice.docToOrdStart; assert docBase == docBases[readerIdx]; - if (currentDocMap != null) { // we have deletes - for (int i = 0; i < currentDocMap.length; i++) { - final int doc = currentDocMap[i]; + if (currentDocMap != null && currentDocMap.hasDeletions()) { // we have deletes + for (int i = 0; i < currentDocMap.maxDoc(); i++) { + final int doc = currentDocMap.get(i); if (doc != -1) { // not deleted final int ord = currentSlice.source.ord(i); // collect ords strictly // increasing Index: lucene/core/src/java/org/apache/lucene/index/MergeState.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MergeState.java (révision 1343109) +++ lucene/core/src/java/org/apache/lucene/index/MergeState.java (copie de travail) @@ -24,6 +24,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.packed.PackedInts; /** Holds common state used during segment merging * @@ -33,16 +34,157 @@ public static class IndexReaderAndLiveDocs { public final AtomicReader reader; public final Bits liveDocs; + public final int numDeletedDocs; - public IndexReaderAndLiveDocs(AtomicReader reader, Bits liveDocs) { + public IndexReaderAndLiveDocs(AtomicReader reader, Bits liveDocs, int numDeletedDocs) { this.reader = reader; this.liveDocs = liveDocs; + this.numDeletedDocs = numDeletedDocs; } } + public static abstract class DocMap { + + public static DocMap build(IndexReaderAndLiveDocs reader) { + final int maxDoc = reader.reader.maxDoc(); + final int numDeletes = reader.numDeletedDocs; + final int numDocs = maxDoc - numDeletes; + if (numDeletes == 0) { + return new NoDelDocMap(maxDoc); + } else if (numDeletes < numDocs) { + return buildDelCountDocmap(maxDoc, numDeletes, reader.liveDocs, PackedInts.FAST); + } else { + return buildDirectDocMap(maxDoc, numDocs, reader.liveDocs, PackedInts.FAST); + } + } + + static DocMap buildDelCountDocmap(int maxDoc, int numDeletes, Bits liveDocs, float acceptableOverheadRatio) { + PackedInts.Mutable numDeletesSoFar = PackedInts.getMutable(maxDoc, + PackedInts.bitsRequired(numDeletes), acceptableOverheadRatio); + int del = 0; + for (int i = 0; i < maxDoc; ++i) { + if (!liveDocs.get(i)) { + ++del; + } + numDeletesSoFar.set(i, del); + } + assert del == numDeletes : "del=" + del + ", numdeletes=" + numDeletes; + return new DelCountDocMap(numDeletesSoFar); + } + + static DocMap buildDirectDocMap(int maxDoc, int numDocs, Bits liveDocs, float acceptableOverheadRatio) { + PackedInts.Mutable docIdsPlusOne = PackedInts.getMutable(maxDoc, + PackedInts.bitsRequired(numDocs), acceptableOverheadRatio); + int del = 0; + for (int i = 0; i < maxDoc; ++i) { + if (liveDocs.get(i)) { + docIdsPlusOne.set(i, i - del + 1); + } else { + ++del; + } + } + assert numDocs + del == maxDoc : "maxDoc=" + maxDoc + ", del=" + del + ", numDocs=" + numDocs; + return new DirectDocMap(docIdsPlusOne, del); + } + + public abstract int get(int docId); + + public abstract int maxDoc(); + + public final int numDocs() { + return maxDoc() - numDeletedDocs(); + } + + public abstract int numDeletedDocs(); + + public boolean hasDeletions() { + return numDeletedDocs() > 0; + } + + } + + private static class NoDelDocMap extends DocMap { + + private final int maxDoc; + + private NoDelDocMap(int maxDoc) { + this.maxDoc = maxDoc; + } + + @Override + public int get(int docId) { + return docId; + } + + @Override + public int maxDoc() { + return maxDoc; + } + + @Override + public int numDeletedDocs() { + return 0; + } + } + + private static class DirectDocMap extends DocMap { + + private final PackedInts.Mutable docIdsPlusOne; + private final int numDeletedDocs; + + private DirectDocMap(PackedInts.Mutable docIdsPlusOne, int numDeletedDocs) { + this.docIdsPlusOne = docIdsPlusOne; + this.numDeletedDocs = numDeletedDocs; + } + + public int get(int docId) { + return (int) docIdsPlusOne.get(docId) - 1; + } + + public int maxDoc() { + return docIdsPlusOne.size(); + } + + public int numDeletedDocs() { + return numDeletedDocs; + } + } + + private static class DelCountDocMap extends DocMap { + + private final PackedInts.Mutable numDeletesSoFar; + + private DelCountDocMap(PackedInts.Mutable numDeletesSoFar) { + this.numDeletesSoFar = numDeletesSoFar; + } + + public int get(int docId) { + if (docId == 0) { + return numDeletesSoFar.get(0) == 0 ? 0 : -1; + } else { + final int numDeletes = (int) numDeletesSoFar.get(docId); + if (numDeletes == numDeletesSoFar.get(docId - 1)) { + // docId is not deleted + return docId - numDeletes; + } else { + return -1; + } + } + } + + public int maxDoc() { + return numDeletesSoFar.size(); + } + + public int numDeletedDocs() { + final int maxDoc = maxDoc(); + return (int) numDeletesSoFar.get(maxDoc - 1); + } + } + public FieldInfos fieldInfos; public List readers; // Readers & liveDocs being merged - public int[][] docMaps; // Maps docIDs around deletions + public DocMap[] docMaps; // Maps docIDs around deletions public int[] docBase; // New docID base per reader public int mergedDocCount; // Total # merged docs public CheckAbort checkAbort; Index: lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (révision 1343109) +++ lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java (copie de travail) @@ -77,7 +77,7 @@ new ReaderUtil.Gather(reader) { @Override protected void add(int base, AtomicReader r) { - mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(r, r.getLiveDocs())); + mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(r, r.getLiveDocs(), r.numDeletedDocs())); } }.run(); } catch (IOException ioe) { @@ -86,8 +86,8 @@ } } - final void add(SegmentReader reader, Bits liveDocs) { - mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(reader, liveDocs)); + final void add(SegmentReader reader, Bits liveDocs, int delCount) { + mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(reader, liveDocs, delCount)); } /** @@ -284,7 +284,7 @@ final int numReaders = mergeState.readers.size(); // Remap docIDs - mergeState.docMaps = new int[numReaders][]; + mergeState.docMaps = new MergeState.DocMap[numReaders]; mergeState.docBase = new int[numReaders]; mergeState.readerPayloadProcessor = new PayloadProcessorProvider.ReaderPayloadProcessor[numReaders]; mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders]; @@ -297,30 +297,9 @@ final MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i); mergeState.docBase[i] = docBase; - final int maxDoc = reader.reader.maxDoc(); - final int docCount; - final Bits liveDocs = reader.liveDocs; - final int[] docMap; - if (liveDocs != null) { - int delCount = 0; - docMap = new int[maxDoc]; - int newDocID = 0; - for(int j=0;j