diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java index 17024d8..83e4817 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Accountable; @@ -659,6 +660,12 @@ public class BlockTermsReader extends FieldsProducer { } @Override + public PostingsEnum topPostings(SimScorer scorer) throws IOException { + decodeMetaData(); + return postingsReader.topPostings(fieldInfo, state, scorer); + } + + @Override public void seekExact(BytesRef target, TermState otherState) { //System.out.println("BTR.seekExact termState target=" + target.utf8ToString() + " " + target + " this=" + this); assert otherState != null && otherState instanceof BlockTermState; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java index f19cd2c..9ed87b5 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.IndexOptions; @@ -127,7 +128,7 @@ public class BlockTermsWriter extends FieldsConsumer implements Closeable { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { @@ -146,7 +147,7 @@ public class BlockTermsWriter extends FieldsConsumer implements Closeable { break; } - termsWriter.write(term, termsEnum); + termsWriter.write(term, termsEnum, norms); } termsWriter.finish(); @@ -232,9 +233,9 @@ public class BlockTermsWriter extends FieldsConsumer implements Closeable { private final BytesRefBuilder lastPrevTerm = new BytesRefBuilder(); - void write(BytesRef text, TermsEnum termsEnum) throws IOException { + void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException { - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms); if (state == null) { // No docs for this term: return; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java index b16bb15..bdc0738 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter; // javadocs import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; @@ -213,7 +214,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { String lastField = null; for(String field : fields) { @@ -233,7 +234,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { if (term == null) { break; } - termsWriter.write(term, termsEnum); + termsWriter.write(term, termsEnum, norms); } termsWriter.finish(); @@ -771,7 +772,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { } /** Writes one term's worth of postings. */ - public void write(BytesRef text, TermsEnum termsEnum) throws IOException { + public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; @@ -780,7 +781,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { } */ - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java index 6c2d2bc..c1e7007 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -205,6 +206,12 @@ final class OrdsIntersectTermsEnum extends TermsEnum { return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags); } + @Override + public PostingsEnum topPostings(SimScorer scorer) throws IOException { + currentFrame.decodeMetaData(); + return fr.parent.postingsReader.topPostings(fr.fieldInfo, currentFrame.termState, scorer); + } + private int getState() { int state = currentFrame.state; for(int idx=0;idx freqNormPairs; + + public CompetitiveFreqNormAccumulator() { + maxFreqs = new int[256]; + Comparator comparator = new Comparator() { + @Override + public int compare(FreqAndNorm o1, FreqAndNorm o2) { + // greater freqs compare greater + int cmp = Integer.compare(o1.freq, o2.freq); + if (cmp == 0) { + // greater norms compare lower + cmp = Long.compareUnsigned(o2.norm, o1.norm); + } + return cmp; + } + }; + freqNormPairs = new TreeSet<>(comparator); + } + + public void clear() { + Arrays.fill(maxFreqs, 0); + dirty = false; + freqNormPairs.clear(); + } + + public static class FreqAndNorm { + public final int freq; + public final long norm; + + public FreqAndNorm(int freq, long norm) { + this.freq = freq; + this.norm = norm; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj instanceof FreqAndNorm == false) { + return false; + } + FreqAndNorm that = (FreqAndNorm) obj; + return freq == that.freq && norm == that.norm; + } + + @Override + public int hashCode() { + int h = getClass().hashCode(); + h = 31 * h + freq; + h = 31 * h + Long.hashCode(norm); + return h; + } + + @Override + public String toString() { + return "{" + freq + "," + norm + "}"; + } + } + + public void add(int freq, long norm) { + if (norm >= Byte.MIN_VALUE && norm <= Byte.MAX_VALUE) { + int index = Byte.toUnsignedInt((byte) norm); + maxFreqs[index] = Math.max(maxFreqs[index], freq); + dirty = true; + } else { + add(new FreqAndNorm(freq, norm)); + } + } + + public void addAll(CompetitiveFreqNormAccumulator acc) { + for (FreqAndNorm entry : acc.getCompetitiveFreqNormPairs()) { + add(entry); + } + } + + public SortedSet getCompetitiveFreqNormPairs() { + if (dirty) { + for (int i = 0; i < maxFreqs.length; ++i) { + if (maxFreqs[i] > 0) { + add(new FreqAndNorm(maxFreqs[i], (byte) i)); + maxFreqs[i] = 0; + } + } + dirty = false; + } + return Collections.unmodifiableSortedSet(freqNormPairs); + } + + private void add(FreqAndNorm newEntry) { + FreqAndNorm next = freqNormPairs.ceiling(newEntry); + if (next == null) { + // nothing is more competitive + freqNormPairs.add(newEntry); + } else if (Long.compareUnsigned(next.norm, newEntry.norm) <= 0) { + // we already have this entry or more competitive entries in the tree + return; + } else { + // some entries have a greater freq but a less competitive norm, so we + // don't know which one will trigger greater scores, still add to the tree + freqNormPairs.add(newEntry); + } + + for (Iterator it = freqNormPairs.headSet(newEntry, false).descendingIterator(); it.hasNext(); ) { + FreqAndNorm entry = it.next(); + if (Long.compareUnsigned(entry.norm, newEntry.norm) >= 0) { + // less competitive + it.remove(); + } else { + // lesser freq but better norm, further entries are not comparable + break; + } + } + } + + @Override + public String toString() { + return getCompetitiveFreqNormPairs().toString(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java index 28bae1d..f4fc9ac 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java @@ -76,14 +76,14 @@ public abstract class FieldsConsumer implements Closeable { * live docs when pulling docs/positions enums. * */ - public abstract void write(Fields fields) throws IOException; + public abstract void write(Fields fields, NormsProducer norms) throws IOException; /** Merges in the fields from the readers in * mergeState. The default implementation skips - * and maps around deleted documents, and calls {@link #write(Fields)}. + * and maps around deleted documents, and calls {@link #write(Fields,NormsProducer)}. * Implementations can override this method for more sophisticated * merging (bulk-byte copying, etc). */ - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { final List fields = new ArrayList<>(); final List slices = new ArrayList<>(); @@ -102,7 +102,7 @@ public abstract class FieldsConsumer implements Closeable { Fields mergedFields = new MappedMultiFields(mergeState, new MultiFields(fields.toArray(Fields.EMPTY_ARRAY), slices.toArray(ReaderSlice.EMPTY_ARRAY))); - write(mergedFields); + write(mergedFields, norms); } // NOTE: strange but necessary so javadocs linting is happy: diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java index c937886..39666c8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java @@ -41,7 +41,7 @@ public abstract class MultiLevelSkipListReader implements Closeable { protected int maxNumberOfSkipLevels; // number of levels in this skip list - private int numberOfSkipLevels; + protected int numberOfSkipLevels; // Expert: defines the number of top skip levels to buffer in memory. // Reducing this number results in less memory usage, but possibly diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java index 207b324..8e090be 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java @@ -53,13 +53,13 @@ import org.apache.lucene.util.MathUtil; public abstract class MultiLevelSkipListWriter { /** number of levels in this skip list */ - protected int numberOfSkipLevels; + protected final int numberOfSkipLevels; /** the skip interval in the list with level = 0 */ - private int skipInterval; + private final int skipInterval; /** skipInterval used for level > 0 */ - private int skipMultiplier; + private final int skipMultiplier; /** for every skip level a different buffer is used */ private RAMOutputStream[] skipBuffer; @@ -69,6 +69,7 @@ public abstract class MultiLevelSkipListWriter { this.skipInterval = skipInterval; this.skipMultiplier = skipMultiplier; + int numberOfSkipLevels; // calculate the maximum number of skip levels for this document frequency if (df <= skipInterval) { numberOfSkipLevels = 1; @@ -80,6 +81,7 @@ public abstract class MultiLevelSkipListWriter { if (numberOfSkipLevels > maxSkipLevels) { numberOfSkipLevels = maxSkipLevels; } + this.numberOfSkipLevels = numberOfSkipLevels; } /** Creates a {@code MultiLevelSkipListWriter}, where diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java index 56cbab5..1539605 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Accountable; @@ -65,7 +66,17 @@ public abstract class PostingsReaderBase implements Closeable, Accountable { /** Must fully consume state, since after this call that * TermState may be reused. */ public abstract PostingsEnum postings(FieldInfo fieldInfo, BlockTermState state, PostingsEnum reuse, int flags) throws IOException; - + + /** + * Return a {@link PostingsEnum} that optionally knows how to skip based on scores. + * The returned {@link PostingsEnum} must support {@link PostingsEnum#FREQS frequencies}. + * The default implementation falls back to + * {@link #postings(FieldInfo, BlockTermState, PostingsEnum, int) postings(fieldInfo, state, null, PostingsEnum.FREQS}. + */ + public PostingsEnum topPostings(FieldInfo fieldInfo, BlockTermState state, Similarity.SimScorer scorer) throws IOException { + return postings(fieldInfo, state, null, PostingsEnum.FREQS); + } + /** * Checks consistency of this reader. *

diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java index b4f2d4e..48c6027 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java @@ -60,7 +60,7 @@ public abstract class PostingsWriterBase implements Closeable { * FixedBitSet} for every docID written. If no docs * were written, this method should return null, and the * terms dict will skip the term. */ - public abstract BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException; + public abstract BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen, NormsProducer norms) throws IOException; /** * Encode metadata as long[] and byte[]. {@code absolute} controls whether diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java index 1fb83b9..98bdd91 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -74,7 +75,7 @@ public abstract class PushPostingsWriterBase extends PostingsWriterBase { /** Start a new term. Note that a matching call to {@link * #finishTerm(BlockTermState)} is done, only if the term has at least one * document. */ - public abstract void startTerm() throws IOException; + public abstract void startTerm(NumericDocValues norms) throws IOException; /** Finishes the current term. The provided {@link * BlockTermState} contains the term's summary statistics, @@ -117,8 +118,14 @@ public abstract class PushPostingsWriterBase extends PostingsWriterBase { } @Override - public final BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException { - startTerm(); + public final BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen, NormsProducer norms) throws IOException { + NumericDocValues normValues; + if (fieldInfo.hasNorms() == false) { + normValues = null; + } else { + normValues = norms.getNorms(fieldInfo); + } + startTerm(normValues); postingsEnum = termsEnum.postings(postingsEnum, enumFlags); assert postingsEnum != null; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java index bdacc22..ec3f6e6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -315,7 +316,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { //if (DEBUG) System.out.println("\nBTTW.write seg=" + segment); String lastField = null; @@ -340,7 +341,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } //if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" + brToString(term)); - termsWriter.write(term, termsEnum); + termsWriter.write(term, termsEnum, norms); } termsWriter.finish(); @@ -852,7 +853,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } /** Writes one term's worth of postings. */ - public void write(BytesRef text, TermsEnum termsEnum) throws IOException { + public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; @@ -861,7 +862,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } */ - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms); if (state != null) { assert state.docFreq != 0; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java index 19e56a4..99f6787 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -235,6 +236,12 @@ final class IntersectTermsEnum extends TermsEnum { return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags); } + @Override + public PostingsEnum topPostings(SimScorer scorer) throws IOException { + currentFrame.decodeMetaData(); + return fr.parent.postingsReader.topPostings(fr.fieldInfo, currentFrame.termState, scorer); + } + private int getState() { int state = currentFrame.state; for(int idx=0;idx= minCompetitiveScore) { + break; + } + + if (nextSkipDoc == NO_MORE_DOCS) { + return doc = NO_MORE_DOCS; + } + + // always plus one to fix the result, since skip position in Lucene50SkipReader + // is a little different from MultiLevelSkipListReader + int newTarget = skipper.skipToScore(minCompetitiveScore) + 1; + if (newTarget <= target) { + skipper.skipToScore(minCompetitiveScore); + } + assert newTarget > target; + target = newTarget; + } while (target > nextSkipDoc); + } + + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } + if (docBufferUpto == BLOCK_SIZE) { + refillDocs(); + } + + // Now scan... this is an inlined/pared down version + // of nextDoc(): + while (true) { + accum += docDeltaBuffer[docBufferUpto]; + docUpto++; + + if (accum >= target) { + break; + } + docBufferUpto++; + if (docUpto == docFreq) { + return doc = NO_MORE_DOCS; + } + } + + freq = freqBuffer[docBufferUpto]; + docBufferUpto++; + return doc = accum; + } + + @Override + public long cost() { + return docFreq; + } + } + @Override public long ramBytesUsed() { return BASE_RAM_BYTES_USED; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java index 6d24a4c..4c87981 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java @@ -31,12 +31,14 @@ import java.io.IOException; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator; import org.apache.lucene.codecs.PushPostingsWriterBase; import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; @@ -96,7 +98,12 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { private final ForUtil forUtil; private final Lucene50SkipWriter skipWriter; - + + private boolean fieldHasNorms; + private NumericDocValues norms; + private CompetitiveFreqNormAccumulator competitiveFreqNormAccumulator; + private int maxFreq; + /** Creates a postings writer */ public Lucene50PostingsWriter(SegmentWriteState state) throws IOException { final float acceptableOverheadRatio = PackedInts.COMPACT; @@ -183,8 +190,16 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { @Override public int setField(FieldInfo fieldInfo) { super.setField(fieldInfo); - skipWriter.setField(writePositions, writeOffsets, writePayloads); + skipWriter.setField(writePositions, writeOffsets, writePayloads, fieldInfo.hasNorms()); lastState = emptyState; + fieldHasNorms = fieldInfo.hasNorms(); + if (fieldHasNorms) { + competitiveFreqNormAccumulator = new CompetitiveFreqNormAccumulator(); + maxFreq = -1; + } else { + competitiveFreqNormAccumulator = null; + maxFreq = 0; + } if (writePositions) { if (writePayloads || writeOffsets) { return 3; // doc + pos + pay FP @@ -197,7 +212,7 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { } @Override - public void startTerm() { + public void startTerm(NumericDocValues norms) { docStartFP = docOut.getFilePointer(); if (writePositions) { posStartFP = posOut.getFilePointer(); @@ -208,6 +223,17 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { lastDocID = 0; lastBlockDocID = -1; skipWriter.resetSkip(); + this.norms = norms; + if (fieldHasNorms) { + assert norms != null; + assert competitiveFreqNormAccumulator != null; + assert maxFreq == -1; + competitiveFreqNormAccumulator.clear(); + } else { + assert norms == null; + assert competitiveFreqNormAccumulator == null; + maxFreq = 0; + } } @Override @@ -216,7 +242,13 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { // Should write skip data as well as postings list for // current block. if (lastBlockDocID != -1 && docBufferUpto == 0) { - skipWriter.bufferSkip(lastBlockDocID, docCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockPayloadByteUpto); + skipWriter.bufferSkip(lastBlockDocID, competitiveFreqNormAccumulator, maxFreq, docCount, + lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockPayloadByteUpto); + if (fieldHasNorms) { + competitiveFreqNormAccumulator.clear(); + } else { + maxFreq = 0; + } } final int docDelta = docID - lastDocID; @@ -247,6 +279,22 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { lastDocID = docID; lastPosition = 0; lastStartOffset = 0; + + if (fieldHasNorms) { + boolean found = norms.advanceExact(docID); + long norm; + if (found == false) { + // This can happen if indexing hits a problem after adding a doc to the + // postings but before buffering the norm. Such documents are written + // deleted and will go away on the first merge. + norm = 1L; + } else { + norm = norms.longValue(); + } + competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm); + } else { + maxFreq = Math.max(maxFreq, writeFreqs ? termDocFreq : 1); + } } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java new file mode 100644 index 0000000..bdd0206 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50ScoreSkipReader.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene50; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; + +import org.apache.lucene.search.similarities.Similarity.SimScorer; +import org.apache.lucene.store.IndexInput; + +final class Lucene50ScoreSkipReader extends Lucene50SkipReader { + + private final SimScorer scorer; + private final float[] maxScore; + private final float globalMaxScore; + + public Lucene50ScoreSkipReader(int version, IndexInput skipStream, int maxSkipLevels, + boolean hasPos, boolean hasOffsets, boolean hasPayloads, SimScorer scorer) { + super(version, skipStream, maxSkipLevels, hasPos, hasOffsets, hasPayloads); + if (version < Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) { + throw new IllegalStateException("Cannot skip based on scores if impacts are not indexed"); + } + this.scorer = Objects.requireNonNull(scorer); + this.maxScore = new float[maxSkipLevels]; + this.globalMaxScore = scorer.maxScore(Float.POSITIVE_INFINITY); + } + + @Override + public void init(long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df) throws IOException { + super.init(skipPointer, docBasePointer, posBasePointer, payBasePointer, df); + Arrays.fill(maxScore, globalMaxScore); + } + + float currentBlockMaxScore() { + return maxScore[0]; + } + + public int skipToScore(float minScore) throws IOException { + int level = -1; + while (level < numberOfSkipLevels - 1 && maxScore[level + 1] < minScore) { + level++; + } + if (level == -1) { + return -1; + } else { + return skipDoc[level]; + } + } + + @Override + protected void readImpacts(int level, IndexInput skipStream) throws IOException { + float maxScore = 0; + if (level > 0) { + long length = skipStream.readVLong(); + long upTo = skipStream.getFilePointer() + length; + int freq = 0; + long norm = 0; + while (skipStream.getFilePointer() < upTo) { + freq += skipStream.readVInt(); + assert freq > 0; + norm += skipStream.readZLong(); + maxScore = Math.max(maxScore, scorer.score(freq, Long.valueOf(norm))); + } + assert skipStream.getFilePointer() == upTo; + } else { + int freq = skipStream.readVInt(); + assert freq > 0; + long norm = skipStream.readZLong(); + maxScore = scorer.score(freq, norm); + // because level 0 uses sloppy data (only the max freq and min norm + // rather than all competitive pairs), we might have better data on + // level 1 + if (numberOfSkipLevels > 1) { + maxScore = Math.min(maxScore, this.maxScore[1]); + } + } + this.maxScore[level] = maxScore; + assert level + 1 == numberOfSkipLevels || this.maxScore[level] <= this.maxScore[level+1]; + } + +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java index 8c037c5..a9d8d70 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java @@ -52,7 +52,8 @@ import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZ * Therefore, we'll trim df before passing it to the interface. see trim(int) * */ -final class Lucene50SkipReader extends MultiLevelSkipListReader { +class Lucene50SkipReader extends MultiLevelSkipListReader { + private final int version; private long docPointer[]; private long posPointer[]; private long payPointer[]; @@ -65,8 +66,11 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { private long lastDocPointer; private int lastPosBufferUpto; - public Lucene50SkipReader(IndexInput skipStream, int maxSkipLevels, boolean hasPos, boolean hasOffsets, boolean hasPayloads) { + public Lucene50SkipReader(int version, + IndexInput skipStream, int maxSkipLevels, + boolean hasPos, boolean hasOffsets, boolean hasPayloads) { super(skipStream, maxSkipLevels, BLOCK_SIZE, 8); + this.version = version; docPointer = new long[maxSkipLevels]; if (hasPos) { posPointer = new long[maxSkipLevels]; @@ -192,6 +196,22 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { payPointer[level] += skipStream.readVLong(); } } + readImpacts(level, skipStream); return delta; } + + // The default impl skips impacts since they are only useful if we have a SimScorer + // to compute the scores that impacts map to. + protected void readImpacts(int level, IndexInput skipStream) throws IOException { + if (version >= Lucene50PostingsFormat.VERSION_IMPACT_SKIP_DATA) { + // The base implementation skips impacts, they are not used + if (level > 0) { // level 0 has a different encoding + skipStream.skipBytes(skipStream.readVLong()); + } else { + skipStream.readVInt(); // freq + skipStream.readZLong(); // norm + } + } + } + } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java index a4556c6..2ae5b09 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java @@ -19,9 +19,13 @@ package org.apache.lucene.codecs.lucene50; import java.io.IOException; import java.util.Arrays; +import java.util.SortedSet; -import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm; import org.apache.lucene.codecs.MultiLevelSkipListWriter; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; /** * Write skip lists with multiple levels, and support skip within block ints. @@ -44,6 +48,7 @@ import org.apache.lucene.codecs.MultiLevelSkipListWriter; * */ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { + private final int maxSkipLevels; private int[] lastSkipDoc; private long[] lastSkipDocPointer; private long[] lastSkipPosPointer; @@ -60,12 +65,16 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { private long curPayPointer; private int curPosBufferUpto; private int curPayloadByteUpto; + private int[] curMaxFreq; + private CompetitiveFreqNormAccumulator[] curCompetitiveFreqNorms; private boolean fieldHasPositions; private boolean fieldHasOffsets; private boolean fieldHasPayloads; + private boolean fieldHasNorms; public Lucene50SkipWriter(int maxSkipLevels, int blockSize, int docCount, IndexOutput docOut, IndexOutput posOut, IndexOutput payOut) { super(blockSize, 8, maxSkipLevels, docCount); + this.maxSkipLevels = maxSkipLevels; this.docOut = docOut; this.posOut = posOut; this.payOut = payOut; @@ -81,10 +90,23 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { } } - public void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) { + public void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads, + boolean fieldHasNorms) { this.fieldHasPositions = fieldHasPositions; this.fieldHasOffsets = fieldHasOffsets; this.fieldHasPayloads = fieldHasPayloads; + this.fieldHasNorms = fieldHasNorms; + if (fieldHasNorms) { + curMaxFreq = null; + curCompetitiveFreqNorms = new CompetitiveFreqNormAccumulator[maxSkipLevels]; + // we do not store competitive (freq,norm) pairs on the first level + for (int i = 0; i < maxSkipLevels; ++i) { + curCompetitiveFreqNorms[i] = new CompetitiveFreqNormAccumulator(); + } + } else { + curMaxFreq = new int[maxSkipLevels]; + curCompetitiveFreqNorms = null; + } } // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the skipper @@ -122,6 +144,13 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { Arrays.fill(lastSkipPayPointer, lastPayFP); } } + if (fieldHasNorms) { + for (CompetitiveFreqNormAccumulator acc : curCompetitiveFreqNorms) { + acc.clear(); + } + } else { + Arrays.fill(curMaxFreq, 0); + } initialized = true; } } @@ -129,7 +158,8 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { /** * Sets the values for the current skip data. */ - public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException { + public void bufferSkip(int doc, CompetitiveFreqNormAccumulator competitiveFreqNorms, int maxFreq, + int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException { initSkip(); this.curDoc = doc; this.curDocPointer = docOut.getFilePointer(); @@ -137,11 +167,21 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { this.curPayPointer = payFP; this.curPosBufferUpto = posBufferUpto; this.curPayloadByteUpto = payloadByteUpto; + if (fieldHasNorms) { + assert maxFreq == -1; + this.curCompetitiveFreqNorms[0].addAll(competitiveFreqNorms); + } else { + assert competitiveFreqNorms == null; + this.curMaxFreq[0] = Math.max(maxFreq, curMaxFreq[0]); + } bufferSkip(numDocs); } - + + private final RAMOutputStream freqNormOut = new RAMOutputStream(); + @Override protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + int delta = curDoc - lastSkipDoc[level]; skipBuffer.writeVInt(delta); @@ -165,5 +205,53 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { lastSkipPayPointer[level] = curPayPointer; } } + + if (fieldHasNorms) { + CompetitiveFreqNormAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level]; + SortedSet freqAndNorms = competitiveFreqNorms.getCompetitiveFreqNormPairs(); + assert freqAndNorms.size() > 0; + if (level + 1 < numberOfSkipLevels) { + curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms); + } + if (level > 0) { + // write all competitive freq/norm pairs + FreqAndNorm previous = new FreqAndNorm(0, 0); + for (FreqAndNorm freqAndNorm : freqAndNorms) { + freqNormOut.writeVInt(freqAndNorm.freq - previous.freq); + freqNormOut.writeZLong(freqAndNorm.norm - previous.norm); + previous = freqAndNorm; + } + skipBuffer.writeVLong(freqNormOut.getFilePointer()); + freqNormOut.writeTo(skipBuffer); + freqNormOut.reset(); + competitiveFreqNorms.clear(); + } else { + // level 0 is written every 256 entries, so we need to make it compact: + // we just write the most competitive freq and norm. Note that this + // means that level 1 might store better upper bounds of the max score + // than level 0, so the reader needs to take that into account. + int bestFreq = freqAndNorms.last().freq; + long bestNorm = freqAndNorms.first().norm; + skipBuffer.writeVInt(bestFreq); + skipBuffer.writeZLong(bestNorm); + } + } else { + if (level + 1 < numberOfSkipLevels) { + curMaxFreq[level + 1] = Math.max(curMaxFreq[level + 1], curMaxFreq[level]); + } + assert curMaxFreq[level] > 0 : curMaxFreq[level]; + // make it compatible with the case that norms are enabled + if (level > 0) { + freqNormOut.writeVInt(curMaxFreq[level]); + freqNormOut.writeZLong(1); + skipBuffer.writeVLong(freqNormOut.getFilePointer()); + freqNormOut.writeTo(skipBuffer); + freqNormOut.reset(); + } else { + skipBuffer.writeVInt(curMaxFreq[level]); + skipBuffer.writeZLong(1L); + } + curMaxFreq[level] = 0; + } } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java index 281b08f..36f0358 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java @@ -34,6 +34,7 @@ import java.util.TreeSet; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -117,7 +118,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { Map formatToGroups = buildFieldsGroupMapping(fields); // Write postings @@ -137,7 +138,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { FieldsConsumer consumer = format.fieldsConsumer(group.state); toClose.add(consumer); - consumer.write(maskedFields); + consumer.write(maskedFields, norms); } success = true; } finally { @@ -148,7 +149,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { } @Override - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { Map formatToGroups = buildFieldsGroupMapping(new MultiFields(mergeState.fieldsProducers, null)); // Merge postings @@ -161,7 +162,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { FieldsConsumer consumer = format.fieldsConsumer(group.state); toClose.add(consumer); - consumer.merge(pfMergeState.apply(group.fields)); + consumer.merge(pfMergeState.apply(group.fields), norms); } success = true; } finally { diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index fd24105..3ade1ed 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -31,6 +31,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.NormsConsumer; import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.document.FieldType; @@ -126,6 +127,7 @@ final class DefaultIndexingChain extends DocConsumer { if (docState.infoStream.isEnabled("IW")) { docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write norms"); } + SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, IOContext.READ, state.segmentSuffix); t0 = System.nanoTime(); writeDocValues(state, sortMap); @@ -159,7 +161,11 @@ final class DefaultIndexingChain extends DocConsumer { } } - termsHash.flush(fieldsToFlush, state, sortMap); + try (NormsProducer norms = readState.fieldInfos.hasNorms() + ? state.segmentInfo.getCodec().normsFormat().normsProducer(readState) + : null) { + termsHash.flush(fieldsToFlush, state, sortMap, norms); + } if (docState.infoStream.isEnabled("IW")) { docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write postings and finish vectors"); } diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index d953f8d..ac70669 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.IOUtils; @@ -78,8 +79,9 @@ final class FreqProxTermsWriter extends TermsHash { } @Override - public void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { - super.flush(fieldsToFlush, state, sortMap); + public void flush(Map fieldsToFlush, final SegmentWriteState state, + Sorter.DocMap sortMap, NormsProducer norms) throws IOException { + super.flush(fieldsToFlush, state, sortMap, norms); // Gather all fields that saw any postings: List allFields = new ArrayList<>(); @@ -105,7 +107,7 @@ final class FreqProxTermsWriter extends TermsHash { FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state); boolean success = false; try { - consumer.write(fields); + consumer.write(fields, norms); success = true; } finally { if (success) { diff --git a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java index fdd32a9..d51ca40 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Scorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -108,4 +109,12 @@ public abstract class PostingsEnum extends DocIdSetIterator { * in the byte[]). */ public abstract BytesRef getPayload() throws IOException; + /** + * Set the minimum competitive score. This gives this postings enum the + * ability to skip over documents that would produce a lower score. + * The default implementation does nothing. + * @see TermsEnum#topPostings + * @see Scorer#setMinCompetitiveScore + */ + public void setMinCompetitiveScore(float minScore) {} } diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java index c67b92d..1bb612c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.NormsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.codecs.StoredFieldsWriter; import org.apache.lucene.codecs.TermVectorsWriter; @@ -109,10 +110,28 @@ final class SegmentMerger { final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, null, context); + final SegmentReadState segmentReadState = new SegmentReadState(directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, + IOContext.READ, segmentWriteState.segmentSuffix); + + if (mergeState.mergeFieldInfos.hasNorms()) { + if (mergeState.infoStream.isEnabled("SM")) { + t0 = System.nanoTime(); + } + mergeNorms(segmentWriteState); + if (mergeState.infoStream.isEnabled("SM")) { + long t1 = System.nanoTime(); + mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge norms [" + numMerged + " docs]"); + } + } + if (mergeState.infoStream.isEnabled("SM")) { t0 = System.nanoTime(); } - mergeTerms(segmentWriteState); + try (NormsProducer normsProducer = mergeState.mergeFieldInfos.hasNorms() + ? codec.normsFormat().normsProducer(segmentReadState) + : null) { + mergeTerms(segmentWriteState, normsProducer); + } if (mergeState.infoStream.isEnabled("SM")) { long t1 = System.nanoTime(); mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge postings [" + numMerged + " docs]"); @@ -139,17 +158,6 @@ final class SegmentMerger { long t1 = System.nanoTime(); mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge points [" + numMerged + " docs]"); } - - if (mergeState.mergeFieldInfos.hasNorms()) { - if (mergeState.infoStream.isEnabled("SM")) { - t0 = System.nanoTime(); - } - mergeNorms(segmentWriteState); - if (mergeState.infoStream.isEnabled("SM")) { - long t1 = System.nanoTime(); - mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge norms [" + numMerged + " docs]"); - } - } if (mergeState.mergeFieldInfos.hasVectors()) { if (mergeState.infoStream.isEnabled("SM")) { @@ -225,9 +233,9 @@ final class SegmentMerger { } } - private void mergeTerms(SegmentWriteState segmentWriteState) throws IOException { + private void mergeTerms(SegmentWriteState segmentWriteState, NormsProducer norms) throws IOException { try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) { - consumer.merge(mergeState); + consumer.merge(mergeState, norms); } } } diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java index dff808e..054ca50 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.codecs.TermVectorsWriter; import org.apache.lucene.search.DocIdSetIterator; @@ -37,8 +38,8 @@ final class SortingTermVectorsConsumer extends TermVectorsConsumer { } @Override - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { - super.flush(fieldsToFlush, state, sortMap); + void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap, NormsProducer norms) throws IOException { + super.flush(fieldsToFlush, state, sortMap, norms); if (tmpDirectory != null) { if (sortMap == null) { // we're lucky the index is already sorted, just rename the temporary file and return diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java index 46dc63c..1ac20dd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.TermVectorsWriter; import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; @@ -53,7 +54,7 @@ class TermVectorsConsumer extends TermsHash { } @Override - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { + void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap, NormsProducer norms) throws IOException { if (writer != null) { int numDocs = state.segmentInfo.maxDoc(); assert numDocs > 0; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java index 4b5755a..4cd3de3 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java @@ -19,6 +19,8 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; @@ -171,6 +173,21 @@ public abstract class TermsEnum implements BytesRefIterator { public abstract PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException; /** + * Return a {@link PostingsEnum} that optionally knows how to skip over + * documents that do not meet a + * {@link PostingsEnum#setMinCompetitiveScore minimum score requirement}. + * The returned {@link PostingsEnum} supports + * {@link PostingsEnum#FREQS frequencies}. + * The default implementation falls back to {@link + * #postings(PostingsEnum, int) postings(null, PostingsEnum.FREQS} meaning + * it doesn't support skipping over low-scoring documents. + * @see Scorer#setMinCompetitiveScore(float) + */ + public PostingsEnum topPostings(SimScorer scorer) throws IOException { + return postings(null, PostingsEnum.FREQS); + } + + /** * Expert: Returns the TermsEnums internal state to position the TermsEnum * without re-seeking the term dictionary. *

@@ -228,7 +245,12 @@ public abstract class TermsEnum implements BytesRefIterator { public PostingsEnum postings(PostingsEnum reuse, int flags) { throw new IllegalStateException("this method should never be called"); } - + + @Override + public PostingsEnum topPostings(SimScorer scorer) throws IOException { + throw new IllegalStateException("this method should never be called"); + } + @Override public BytesRef next() { return null; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java index bede2f8..f420aca 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IntBlockPool; @@ -76,13 +77,14 @@ abstract class TermsHash { bytePool.reset(false, false); } - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { + void flush(Map fieldsToFlush, final SegmentWriteState state, + Sorter.DocMap sortMap, NormsProducer norms) throws IOException { if (nextTermsHash != null) { Map nextChildFields = new HashMap<>(); for (final Map.Entry entry : fieldsToFlush.entrySet()) { nextChildFields.put(entry.getKey(), entry.getValue().nextPerField); } - nextTermsHash.flush(nextChildFields, state, sortMap); + nextTermsHash.flush(nextChildFields, state, sortMap, norms); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index 3fa465d..2c08f4f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -47,21 +47,21 @@ public class TermQuery extends Query { private final Similarity similarity; private final Similarity.SimScorer simScorer; private final TermContext termStates; - private final boolean needsScores; + private final ScoreMode scoreMode; - public TermWeight(IndexSearcher searcher, boolean needsScores, + public TermWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost, TermContext termStates) throws IOException { super(TermQuery.this); - if (needsScores && termStates == null) { + if (scoreMode.needsScores() && termStates == null) { throw new IllegalStateException("termStates are required when scores are needed"); } - this.needsScores = needsScores; + this.scoreMode = scoreMode; this.termStates = termStates; this.similarity = searcher.getSimilarity(); final CollectionStatistics collectionStats; final TermStatistics termStats; - if (needsScores) { + if (scoreMode.needsScores()) { collectionStats = searcher.collectionStatistics(term.field()); termStats = searcher.termStatistics(term, termStates); } else { @@ -98,10 +98,15 @@ public class TermQuery extends Query { .getFieldInfos() .fieldInfo(getTerm().field()) .getIndexOptions(); - PostingsEnum docs = termsEnum.postings(null, needsScores ? PostingsEnum.FREQS : PostingsEnum.NONE); + PostingsEnum docs; + if (scoreMode == ScoreMode.TOP_SCORES) { + docs = termsEnum.topPostings(simScorer); + } else { + docs = termsEnum.postings(null, scoreMode.needsScores() ? PostingsEnum.FREQS : PostingsEnum.NONE); + } assert docs != null; float maxFreq = getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq()); - return new TermScorer(this, docs, new LeafSimScorer(simScorer, context.reader(), needsScores, maxFreq)); + return new TermScorer(this, docs, new LeafSimScorer(simScorer, context.reader(), scoreMode.needsScores(), maxFreq)); } private long getMaxFreq(IndexOptions indexOptions, long ttf, long df) { @@ -220,7 +225,7 @@ public class TermQuery extends Query { termState = this.perReaderTermState; } - return new TermWeight(searcher, scoreMode.needsScores(), boost, termState); + return new TermWeight(searcher, scoreMode, boost, termState); } /** Prints a user-readable version of this query. */ diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java index 653a60e..f9c8a07 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java @@ -59,6 +59,11 @@ final class TermScorer extends Scorer { } @Override + public void setMinCompetitiveScore(float minScore) { + postingsEnum.setMinCompetitiveScore(minScore); + } + + @Override public float score() throws IOException { assert docID() != DocIdSetIterator.NO_MORE_DOCS; return docScorer.score(postingsEnum.docID(), postingsEnum.freq()); diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java new file mode 100644 index 0000000..1870a11 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm; +import org.apache.lucene.util.LuceneTestCase; + +public class TestCompetitiveFreqNormAccumulator extends LuceneTestCase { + + public void testBasics() { + CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator(); + Set expected = new HashSet<>(); + + acc.add(3, 5); + expected.add(new FreqAndNorm(3, 5)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(6, 11); + expected.add(new FreqAndNorm(6, 11)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(10, 13); + expected.add(new FreqAndNorm(10, 13)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(1, 2); + expected.add(new FreqAndNorm(1, 2)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(7, 9); + expected.remove(new FreqAndNorm(6, 11)); + expected.add(new FreqAndNorm(7, 9)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(8, 2); + expected.clear(); + expected.add(new FreqAndNorm(10, 13)); + expected.add(new FreqAndNorm(8, 2)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java index 804f507..84544bc 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.asserting.AssertingCodec; import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval; @@ -407,17 +408,17 @@ public class TestPerFieldPostingsFormat2 extends LuceneTestCase { final FieldsConsumer consumer = delegate.fieldsConsumer(state); return new FieldsConsumer() { @Override - public void write(Fields fields) throws IOException { - consumer.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + consumer.write(fields, norms); } @Override - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { nbMergeCalls++; for (FieldInfo fi : mergeState.mergeFieldInfos) { fieldNames.add(fi.name); } - consumer.merge(mergeState); + consumer.merge(mergeState, norms); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index 4625f73..5a71d05 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -28,6 +29,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; @@ -752,9 +754,65 @@ public class TestCodecs extends LuceneTestCase { Arrays.sort(fields); FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state); + NormsProducer fakeNorms = new NormsProducer() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() throws IOException {} + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + return new NumericDocValues() { + + int doc = -1; + + @Override + public int nextDoc() throws IOException { + return advance(doc + 1); + } + + @Override + public int docID() { + return doc; + } + + @Override + public long cost() { + return si.maxDoc(); + } + + @Override + public int advance(int target) throws IOException { + if (target >= si.maxDoc()) { + return doc = NO_MORE_DOCS; + } else { + return doc = target; + } + } + + @Override + public boolean advanceExact(int target) throws IOException { + doc = target; + return true; + } + + @Override + public long longValue() throws IOException { + return 1; + } + }; + } + + @Override + public void checkIntegrity() throws IOException {} + }; boolean success = false; try { - consumer.write(new DataFields(fields)); + consumer.write(new DataFields(fields), fakeNorms); success = true; } finally { if (success) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java index fc643d2..30e1980 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PushPostingsWriterBase; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; @@ -78,7 +79,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase { } @Override - public void startTerm() { + public void startTerm(NumericDocValues norms) { lastDocID = -1; } diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java index e9187af..d8abc6a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter; import org.apache.lucene.index.FieldInfo; @@ -221,7 +222,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { String lastField = null; for(String field : fields) { @@ -730,7 +731,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer { /** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum) throws IOException { - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, null); // TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush: if (state != null && ((IDVersionPostingsWriter) postingsWriter).lastDocID != -1) { assert state.docFreq != 0; diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java index 9df9d60..5ab16a9 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -44,7 +45,7 @@ import static org.apache.lucene.search.suggest.document.CompletionPostingsFormat /** *

- * Weighted FSTs for any indexed {@link SuggestField} is built on {@link #write(Fields)}. + * Weighted FSTs for any indexed {@link SuggestField} is built on {@link #write(Fields,NormsProducer)}. * A weighted FST maps the analyzed forms of a field to its * surface form and document id. FSTs are stored in the CompletionDictionary (.lkp). *

@@ -80,8 +81,8 @@ final class CompletionFieldsConsumer extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { - delegateFieldsConsumer.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + delegateFieldsConsumer.write(fields, norms); for (String field : fields) { CompletionTermWriter termWriter = new CompletionTermWriter(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java index a89b508..e71903d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java @@ -22,6 +22,7 @@ import java.util.Iterator; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.AssertingLeafReader; import org.apache.lucene.index.FieldInfo; @@ -133,8 +134,8 @@ public final class AssertingPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { - in.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + in.write(fields, norms); // TODO: more asserts? can we somehow run a // "limited" CheckIndex here??? Or ... can we improve diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java index 2ca1bc7..00e168b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java @@ -21,6 +21,7 @@ import java.util.Random; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.Fields; import org.apache.lucene.index.SegmentReadState; @@ -61,11 +62,11 @@ class CrankyPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { if (random.nextInt(100) == 0) { throw new IOException("Fake IOException from FieldsConsumer.write()"); } - delegate.write(fields); + delegate.write(fields, norms); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java index 4b85f13..24c94a3 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.PostingsEnum; @@ -227,7 +228,7 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { Terms terms = fields.terms(field); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java index ab92946..f5b5223 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java @@ -334,8 +334,30 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { SegmentReadState readState = new SegmentReadState(dir, segmentInfo, fieldInfos, IOContext.READ); // PostingsFormat + NormsProducer fakeNorms = new NormsProducer() { + + @Override + public void close() throws IOException {} + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + if (field.hasNorms() == false) { + return null; + } + return oneDocReader.getNormValues(field.name); + } + + @Override + public void checkIntegrity() throws IOException {} + + }; try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState)) { - consumer.write(MultiFields.getFields(oneDocReader)); + consumer.write(MultiFields.getFields(oneDocReader), fakeNorms); IOUtils.close(consumer); IOUtils.close(consumer); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java index f69ca55..51b418e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java @@ -32,6 +32,7 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -398,8 +399,8 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest return new FieldsConsumer() { @Override - public void write(Fields fields) throws IOException { - fieldsConsumer.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + fieldsConsumer.write(fields, norms); boolean isMerge = state.context.context == IOContext.Context.MERGE; diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java index d5eb105..fe80943 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/RandomPostingsTester.java @@ -36,6 +36,8 @@ import java.util.TreeMap; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; @@ -653,10 +655,69 @@ public class RandomPostingsTester { Fields seedFields = new SeedFields(fields, newFieldInfos, maxAllowed, allowPayloads); + NormsProducer fakeNorms = new NormsProducer() { + + @Override + public void close() throws IOException {} + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + if (newFieldInfos.fieldInfo(field.number).hasNorms()) { + return new NumericDocValues() { + + int doc = -1; + + @Override + public int nextDoc() throws IOException { + if (++doc == segmentInfo.maxDoc()) { + return doc = NO_MORE_DOCS; + } + return doc; + } + + @Override + public int docID() { + return doc; + } + + @Override + public long cost() { + return segmentInfo.maxDoc(); + } + + @Override + public int advance(int target) throws IOException { + return doc = target >= segmentInfo.maxDoc() ? DocIdSetIterator.NO_MORE_DOCS : target; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return true; + } + + @Override + public long longValue() throws IOException { + return doc & 0x0f; + } + }; + } else { + return null; + } + } + + @Override + public void checkIntegrity() throws IOException {} + + }; FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState); boolean success = false; try { - consumer.write(seedFields); + consumer.write(seedFields, fakeNorms); success = true; } finally { if (success) {