diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java index 17024d8..4fee3d2 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Accountable; @@ -651,11 +652,11 @@ public class BlockTermsReader extends FieldsProducer { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { //System.out.println("BTR.docs this=" + this); decodeMetaData(); //System.out.println("BTR.docs: state.docFreq=" + state.docFreq); - return postingsReader.postings(fieldInfo, state, reuse, flags); + return postingsReader.postings(fieldInfo, state, reuse, scorer, flags); } @Override diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java index f19cd2c..ac1961d 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsWriter.java @@ -26,6 +26,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.IndexOptions; @@ -127,7 +128,7 @@ public class BlockTermsWriter extends FieldsConsumer implements Closeable { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { @@ -234,7 +235,7 @@ public class BlockTermsWriter extends FieldsConsumer implements Closeable { void write(BytesRef text, TermsEnum termsEnum) throws IOException { - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, null); if (state == null) { // No docs for this term: return; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java index b16bb15..f8dc233 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter; // javadocs import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; @@ -213,7 +214,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { String lastField = null; for(String field : fields) { @@ -780,7 +781,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer { } */ - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, null); if (state != null) { assert state.docFreq != 0; assert fieldInfo.getIndexOptions() == IndexOptions.DOCS || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java index 6c2d2bc..780aaf7 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -200,9 +201,9 @@ final class OrdsIntersectTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { currentFrame.decodeMetaData(); - return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags); + return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, scorer, flags); } private int getState() { diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java index 9a9d871..d7c12c7 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java @@ -28,6 +28,7 @@ import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; @@ -921,7 +922,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); @@ -930,7 +931,7 @@ public final class OrdsSegmentTermsEnum extends TermsEnum { //if (DEBUG) { //System.out.println(" state=" + currentFrame.state); //} - return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, flags); + return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, scorer, flags); } @Override diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java index b864bf0..20d2846 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java @@ -29,6 +29,7 @@ import java.util.Map.Entry; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.bloom.FuzzySet.ContainsResult; import org.apache.lucene.index.PostingsEnum; @@ -39,6 +40,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; @@ -366,9 +368,9 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { - return delegate().postings(reuse, flags); + return delegate().postings(reuse, scorer, flags); } } @@ -416,7 +418,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { // Delegate must write first: it may have opened files // on creating the class @@ -424,7 +426,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat { // close them; alternatively, if we delayed pulling // the fields consumer until here, we could do it // afterwards: - delegateFieldsConsumer.write(fields); + delegateFieldsConsumer.write(fields, norms); for(String field : fields) { Terms terms = fields.terms(field); @@ -452,7 +454,7 @@ public final class BloomFilteringPostingsFormat extends PostingsFormat { bloomFilters.put(fieldInfo, bloomFilter); } // Make sure there's at least one doc for this term: - postingsEnum = termsEnum.postings(postingsEnum, 0); + postingsEnum = termsEnum.postings(postingsEnum, null, 0); if (postingsEnum.nextDoc() != PostingsEnum.NO_MORE_DOCS) { bloomFilter.addValue(term); } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java index 00f25cf..01f12aa 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.Accountable; @@ -355,7 +356,7 @@ public final class DirectPostingsFormat extends PostingsFormat { termOffsets[count+1] = termOffset; if (hasPos) { - docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); } else { postingsEnum = termsEnum.postings(postingsEnum); } @@ -843,7 +844,7 @@ public final class DirectPostingsFormat extends PostingsFormat { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { // TODO: implement reuse // it's hairy! @@ -1451,7 +1452,7 @@ public final class DirectPostingsFormat extends PostingsFormat { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { // TODO: implement reuse // it's hairy! diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java index 5ba4c5f..42c545c 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java @@ -41,6 +41,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; @@ -427,9 +428,9 @@ public class FSTOrdTermsReader extends FieldsProducer { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { decodeMetaData(); - return postingsReader.postings(fieldInfo, state, reuse, flags); + return postingsReader.postings(fieldInfo, state, reuse, scorer, flags); } // TODO: this can be achieved by making use of Util.getByOutput() diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java index cbe6583..b59d41d 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -186,7 +187,7 @@ public class FSTOrdTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { Terms terms = fields.terms(field); if (terms == null) { @@ -205,7 +206,7 @@ public class FSTOrdTermsWriter extends FieldsConsumer { if (term == null) { break; } - BlockTermState termState = postingsWriter.writeTerm(term, termsEnum, docsSeen); + BlockTermState termState = postingsWriter.writeTerm(term, termsEnum, docsSeen, norms); if (termState != null) { termsWriter.finishTerm(term, termState); sumTotalTermFreq += termState.totalTermFreq; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index 8dda05c..728a10b 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -41,6 +41,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Accountable; @@ -293,9 +294,9 @@ public class FSTTermsReader extends FieldsProducer { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { decodeMetaData(); - return postingsReader.postings(fieldInfo, state, reuse, flags); + return postingsReader.postings(fieldInfo, state, reuse, scorer, flags); } @Override diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 8284d74..8e55d41 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.FieldInfo; @@ -158,7 +159,7 @@ public class FSTTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { Terms terms = fields.terms(field); if (terms == null) { @@ -179,7 +180,7 @@ public class FSTTermsWriter extends FieldsConsumer { break; } - BlockTermState termState = postingsWriter.writeTerm(term, termsEnum, docsSeen); + BlockTermState termState = postingsWriter.writeTerm(term, termsEnum, docsSeen, norms); if (termState != null) { termsWriter.finishTerm(term, termState); sumTotalTermFreq += termState.totalTermFreq; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java index b81e56e..d3a8a29 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java @@ -30,6 +30,7 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.*; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; @@ -866,7 +867,7 @@ class MemoryDocValuesProducer extends DocValuesProducer { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { throw new UnsupportedOperationException(); } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java index 21983c6..d9a1955 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java @@ -27,6 +27,7 @@ import java.util.TreeMap; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.CorruptIndexException; @@ -40,6 +41,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IOContext; @@ -287,7 +289,7 @@ public final class MemoryPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { Terms terms = fields.terms(field); @@ -340,10 +342,10 @@ public final class MemoryPostingsFormat extends PostingsFormat { termsWriter.postingsWriter.reset(); if (writePositions) { - posEnum = termsEnum.postings(posEnum, enumFlags); + posEnum = termsEnum.postings(posEnum, null, enumFlags); postingsEnum = posEnum; } else { - postingsEnum = termsEnum.postings(postingsEnum, enumFlags); + postingsEnum = termsEnum.postings(postingsEnum, null, enumFlags); posEnum = null; } @@ -780,7 +782,7 @@ public final class MemoryPostingsFormat extends PostingsFormat { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { // TODO: the logic of which enum impl to choose should be refactored to be simpler... boolean hasPositions = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java index f5504b3..eac3e1b 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java @@ -34,6 +34,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; @@ -206,7 +207,7 @@ class SimpleTextFieldsReader extends FieldsProducer { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { boolean hasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java index 2023552..ef9e705 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java @@ -20,6 +20,7 @@ package org.apache.lucene.codecs.simpletext; import java.io.IOException; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -56,7 +57,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { write(writeState.fieldInfos, fields); } @@ -103,7 +104,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { break; } - postingsEnum = termsEnum.postings(postingsEnum, flags); + postingsEnum = termsEnum.postings(postingsEnum, null, flags); assert postingsEnum != null: "termsEnum=" + termsEnum + " hasPos=" + hasPositions + " flags=" + flags; diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java index 25f2a4d..f450c74 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedChecksumIndexInput; import org.apache.lucene.store.ChecksumIndexInput; @@ -392,7 +393,7 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { SimpleTVPostings postings = current.getValue(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java new file mode 100644 index 0000000..87a681d --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveFreqNormAccumulator.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +public final class CompetitiveFreqNormAccumulator { + + // We speed up accumulation for common norm values by first computing + // the max freq for all norms in -128..127 + private final int[] maxFreqs; + private boolean dirty; + private final TreeSet freqNormPairs; + + public CompetitiveFreqNormAccumulator() { + maxFreqs = new int[256]; + Comparator comparator = new Comparator() { + @Override + public int compare(FreqAndNorm o1, FreqAndNorm o2) { + // greater freqs compare greater + int cmp = Integer.compare(o1.freq, o2.freq); + if (cmp == 0) { + // greater norms compare lower + cmp = Long.compareUnsigned(o2.norm, o1.norm); + } + return cmp; + } + }; + freqNormPairs = new TreeSet<>(comparator); + } + + public void clear() { + Arrays.fill(maxFreqs, 0); + dirty = false; + freqNormPairs.clear(); + } + + public static class FreqAndNorm { + public final int freq; + public final long norm; + + public FreqAndNorm(int freq, long norm) { + this.freq = freq; + this.norm = norm; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj instanceof FreqAndNorm == false) { + return false; + } + FreqAndNorm that = (FreqAndNorm) obj; + return freq == that.freq && norm == that.norm; + } + + @Override + public int hashCode() { + int h = getClass().hashCode(); + h = 31 * h + freq; + h = 31 * h + Long.hashCode(norm); + return h; + } + + @Override + public String toString() { + return "{" + freq + "," + norm + "}"; + } + } + + public void add(int freq, long norm) { + if (norm >= Byte.MIN_VALUE && norm <= Byte.MAX_VALUE) { + int index = Byte.toUnsignedInt((byte) norm); + maxFreqs[index] = Math.max(maxFreqs[index], freq); + dirty = true; + } else { + add(new FreqAndNorm(freq, norm)); + } + } + + public void addAll(CompetitiveFreqNormAccumulator acc) { + for (FreqAndNorm entry : acc.getCompetitiveFreqNormPairs()) { + add(entry); + } + } + + public Set getCompetitiveFreqNormPairs() { + if (dirty) { + for (int i = 0; i < maxFreqs.length; ++i) { + if (maxFreqs[i] > 0) { + add(new FreqAndNorm(maxFreqs[i], (byte) i)); + maxFreqs[i] = 0; + } + } + dirty = false; + } + return Collections.unmodifiableSet(freqNormPairs); + } + + private void add(FreqAndNorm newEntry) { + FreqAndNorm next = freqNormPairs.ceiling(newEntry); + if (next == null) { + // nothing is more competitive + freqNormPairs.add(newEntry); + } else if (Long.compareUnsigned(next.norm, newEntry.norm) <= 0) { + // we already have this entry or more competitive entries in the tree + return; + } else { + // some entries have a greater freq but a less competitive norm, so we + // don't know which one will trigger greater scores, still add to the tree + freqNormPairs.add(newEntry); + } + + for (Iterator it = freqNormPairs.headSet(newEntry, false).descendingIterator(); it.hasNext(); ) { + FreqAndNorm entry = it.next(); + if (Long.compareUnsigned(entry.norm, newEntry.norm) >= 0) { + // less competitive + it.remove(); + } else { + // lesser freq but better norm, further entries are not comparable + break; + } + } + } + + @Override + public String toString() { + return getCompetitiveFreqNormPairs().toString(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java index 28bae1d..f4fc9ac 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java @@ -76,14 +76,14 @@ public abstract class FieldsConsumer implements Closeable { * live docs when pulling docs/positions enums. * */ - public abstract void write(Fields fields) throws IOException; + public abstract void write(Fields fields, NormsProducer norms) throws IOException; /** Merges in the fields from the readers in * mergeState. The default implementation skips - * and maps around deleted documents, and calls {@link #write(Fields)}. + * and maps around deleted documents, and calls {@link #write(Fields,NormsProducer)}. * Implementations can override this method for more sophisticated * merging (bulk-byte copying, etc). */ - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { final List fields = new ArrayList<>(); final List slices = new ArrayList<>(); @@ -102,7 +102,7 @@ public abstract class FieldsConsumer implements Closeable { Fields mergedFields = new MappedMultiFields(mergeState, new MultiFields(fields.toArray(Fields.EMPTY_ARRAY), slices.toArray(ReaderSlice.EMPTY_ARRAY))); - write(mergedFields); + write(mergedFields, norms); } // NOTE: strange but necessary so javadocs linting is happy: diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java index c937886..39666c8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java @@ -41,7 +41,7 @@ public abstract class MultiLevelSkipListReader implements Closeable { protected int maxNumberOfSkipLevels; // number of levels in this skip list - private int numberOfSkipLevels; + protected int numberOfSkipLevels; // Expert: defines the number of top skip levels to buffer in memory. // Reducing this number results in less memory usage, but possibly diff --git a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java index 207b324..8e090be 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java @@ -53,13 +53,13 @@ import org.apache.lucene.util.MathUtil; public abstract class MultiLevelSkipListWriter { /** number of levels in this skip list */ - protected int numberOfSkipLevels; + protected final int numberOfSkipLevels; /** the skip interval in the list with level = 0 */ - private int skipInterval; + private final int skipInterval; /** skipInterval used for level > 0 */ - private int skipMultiplier; + private final int skipMultiplier; /** for every skip level a different buffer is used */ private RAMOutputStream[] skipBuffer; @@ -69,6 +69,7 @@ public abstract class MultiLevelSkipListWriter { this.skipInterval = skipInterval; this.skipMultiplier = skipMultiplier; + int numberOfSkipLevels; // calculate the maximum number of skip levels for this document frequency if (df <= skipInterval) { numberOfSkipLevels = 1; @@ -80,6 +81,7 @@ public abstract class MultiLevelSkipListWriter { if (numberOfSkipLevels > maxSkipLevels) { numberOfSkipLevels = maxSkipLevels; } + this.numberOfSkipLevels = numberOfSkipLevels; } /** Creates a {@code MultiLevelSkipListWriter}, where diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java index 56cbab5..5fe70f9 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java @@ -23,6 +23,7 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Accountable; @@ -64,7 +65,7 @@ public abstract class PostingsReaderBase implements Closeable, Accountable { /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract PostingsEnum postings(FieldInfo fieldInfo, BlockTermState state, PostingsEnum reuse, int flags) throws IOException; + public abstract PostingsEnum postings(FieldInfo fieldInfo, BlockTermState state, PostingsEnum reuse, SimScorer scorer, int flags) throws IOException; /** * Checks consistency of this reader. diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java index b4f2d4e..48c6027 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java @@ -60,7 +60,7 @@ public abstract class PostingsWriterBase implements Closeable { * FixedBitSet} for every docID written. If no docs * were written, this method should return null, and the * terms dict will skip the term. */ - public abstract BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException; + public abstract BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen, NormsProducer norms) throws IOException; /** * Encode metadata as long[] and byte[]. {@code absolute} controls whether diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java index 1fb83b9..061ba04 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java @@ -22,6 +22,7 @@ import java.io.IOException; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; @@ -74,7 +75,7 @@ public abstract class PushPostingsWriterBase extends PostingsWriterBase { /** Start a new term. Note that a matching call to {@link * #finishTerm(BlockTermState)} is done, only if the term has at least one * document. */ - public abstract void startTerm() throws IOException; + public abstract void startTerm(NumericDocValues norms) throws IOException; /** Finishes the current term. The provided {@link * BlockTermState} contains the term's summary statistics, @@ -117,9 +118,15 @@ public abstract class PushPostingsWriterBase extends PostingsWriterBase { } @Override - public final BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException { - startTerm(); - postingsEnum = termsEnum.postings(postingsEnum, enumFlags); + public final BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen, NormsProducer norms) throws IOException { + NumericDocValues normValues; + if (fieldInfo.hasNorms() == false) { + normValues = null; + } else { + normValues = norms.getNorms(fieldInfo); + } + startTerm(normValues); + postingsEnum = termsEnum.postings(postingsEnum, null, enumFlags); assert postingsEnum != null; int docFreq = 0; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java index b84065a..4ab66a4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java @@ -295,7 +295,7 @@ public abstract class TermVectorsWriter implements Closeable { startTerm(termsEnum.term(), freq); if (hasPositions || hasOffsets) { - docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS); assert docsAndPositionsEnum != null; final int docID = docsAndPositionsEnum.nextDoc(); diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BitSetTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BitSetTermsEnum.java index ffc182f..e5036e0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BitSetTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BitSetTermsEnum.java @@ -20,6 +20,7 @@ package org.apache.lucene.codecs.blocktree; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.BytesRef; @@ -72,7 +73,7 @@ class BitSetTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { if (flags != PostingsEnum.NONE) { // We only work with DOCS_ONLY fields return null; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java index bdacc22..ec3f6e6 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -315,7 +316,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { //if (DEBUG) System.out.println("\nBTTW.write seg=" + segment); String lastField = null; @@ -340,7 +341,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } //if (DEBUG) System.out.println("write field=" + fieldInfo.name + " term=" + brToString(term)); - termsWriter.write(term, termsEnum); + termsWriter.write(term, termsEnum, norms); } termsWriter.finish(); @@ -852,7 +853,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } /** Writes one term's worth of postings. */ - public void write(BytesRef text, TermsEnum termsEnum) throws IOException { + public void write(BytesRef text, TermsEnum termsEnum, NormsProducer norms) throws IOException { /* if (DEBUG) { int[] tmp = new int[lastTerm.length]; @@ -861,7 +862,7 @@ public final class BlockTreeTermsWriter extends FieldsConsumer { } */ - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, norms); if (state != null) { assert state.docFreq != 0; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java index 19e56a4..6e642ec 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -230,9 +231,9 @@ final class IntersectTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { currentFrame.decodeMetaData(); - return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, flags); + return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.termState, reuse, scorer, flags); } private int getState() { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java index 73c32bb..4b6ecd7 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; @@ -990,7 +991,7 @@ final class SegmentTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); @@ -999,7 +1000,7 @@ final class SegmentTermsEnum extends TermsEnum { //if (DEBUG) { //System.out.println(" state=" + currentFrame.state); //} - return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, flags); + return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, scorer, flags); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java index f5318ba..50ef786 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java @@ -35,6 +35,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; @@ -930,7 +931,7 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem } @Override - public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public final PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { final TVPostingsEnum docsEnum; if (reuse != null && reuse instanceof TVPostingsEnum) { docsEnum = (TVPostingsEnum) reuse; diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java index 0dde774..d7347d0 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; @@ -191,7 +192,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { } @Override - public PostingsEnum postings(FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { boolean indexHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; boolean indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; @@ -207,7 +208,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { } else { docsEnum = new BlockDocsEnum(fieldInfo); } - return docsEnum.reset((IntBlockTermState) termState, flags); + return docsEnum.reset((IntBlockTermState) termState, scorer, flags); } else if ((indexHasOffsets == false || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) && (indexHasPayloads == false || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) { BlockPostingsEnum docsAndPositionsEnum; @@ -252,6 +253,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { final boolean indexHasPos; final boolean indexHasOffsets; final boolean indexHasPayloads; + final boolean fieldHasNorms; private int docFreq; // number of docs in this posting list private long totalTermFreq; // sum of freqs in this posting list (or docFreq when omitted) @@ -275,6 +277,8 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { private boolean needsFreq; // true if the caller actually needs frequencies private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1 + private SimScorer scorer; + public BlockDocsEnum(FieldInfo fieldInfo) throws IOException { this.startDocIn = Lucene50PostingsReader.this.docIn; this.docIn = null; @@ -282,6 +286,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { indexHasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; indexHasPayloads = fieldInfo.hasPayloads(); + fieldHasNorms = fieldInfo.hasNorms(); encoded = new byte[MAX_ENCODED_SIZE]; } @@ -289,10 +294,10 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { return docIn == startDocIn && indexHasFreq == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0) && indexHasPos == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) && - indexHasPayloads == fieldInfo.hasPayloads(); + indexHasPayloads == fieldInfo.hasPayloads() && fieldHasNorms == fieldInfo.hasNorms(); } - public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException { + public PostingsEnum reset(IntBlockTermState termState, SimScorer scorer, int flags) throws IOException { docFreq = termState.docFreq; totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq; docTermStartFP = termState.docStartFP; @@ -316,6 +321,10 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block docBufferUpto = BLOCK_SIZE; skipped = false; + this.scorer = scorer; + if (skipper != null) { + skipper.setScorer(scorer); + } return this; } @@ -405,7 +414,9 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { MAX_SKIP_LEVELS, indexHasPos, indexHasOffsets, - indexHasPayloads); + indexHasPayloads, + fieldHasNorms, + scorer); } if (!skipped) { @@ -460,7 +471,36 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { docBufferUpto++; return doc = accum; } - + + @Override + public int advance(int target, float score) throws IOException { + if (docFreq > BLOCK_SIZE) { + + if (skipper == null) { + // Lazy init: first time this enum has ever been used for skipping + skipper = new Lucene50SkipReader(docIn.clone(), + MAX_SKIP_LEVELS, + indexHasPos, + indexHasOffsets, + indexHasPayloads, + fieldHasNorms, + scorer); + } + + if (!skipped) { + assert skipOffset != -1; + // This is the first time this enum has skipped + // since reset() was called; load the skip data: + skipper.init(docTermStartFP+skipOffset, docTermStartFP, 0, 0, docFreq); + skipped = true; + } + + int nextScoreSkipDoc = skipper.skipToScore(score); + target = Math.max(target, nextScoreSkipDoc + 1); + } + return advance(target); + } + @Override public long cost() { return docFreq; @@ -489,6 +529,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { final boolean indexHasOffsets; final boolean indexHasPayloads; + final boolean fieldHasNorms; private int docFreq; // number of docs in this posting list private long totalTermFreq; // number of positions in this posting list @@ -537,12 +578,13 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { encoded = new byte[MAX_ENCODED_SIZE]; indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; indexHasPayloads = fieldInfo.hasPayloads(); + fieldHasNorms = fieldInfo.hasNorms(); } public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { return docIn == startDocIn && indexHasOffsets == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) && - indexHasPayloads == fieldInfo.hasPayloads(); + indexHasPayloads == fieldInfo.hasPayloads() && fieldHasNorms == fieldInfo.hasNorms(); } public PostingsEnum reset(IntBlockTermState termState) throws IOException { @@ -670,7 +712,9 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { MAX_SKIP_LEVELS, true, indexHasOffsets, - indexHasPayloads); + indexHasPayloads, + fieldHasNorms, + null); } if (!skipped) { @@ -835,6 +879,7 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { final boolean indexHasOffsets; final boolean indexHasPayloads; + final boolean fieldHasNorms; private int docFreq; // number of docs in this posting list private long totalTermFreq; // number of positions in this posting list @@ -909,12 +954,13 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { payloadBytes = null; payload = null; } + fieldHasNorms = fieldInfo.hasNorms(); } public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { return docIn == startDocIn && indexHasOffsets == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) && - indexHasPayloads == fieldInfo.hasPayloads(); + indexHasPayloads == fieldInfo.hasPayloads() && fieldHasNorms == fieldInfo.hasNorms(); } public EverythingEnum reset(IntBlockTermState termState, int flags) throws IOException { @@ -1086,7 +1132,9 @@ public final class Lucene50PostingsReader extends PostingsReaderBase { MAX_SKIP_LEVELS, true, indexHasOffsets, - indexHasPayloads); + indexHasPayloads, + fieldHasNorms, + null); } if (!skipped) { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java index 6d24a4c..ced0299 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsWriter.java @@ -31,12 +31,14 @@ import java.io.IOException; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator; import org.apache.lucene.codecs.PushPostingsWriterBase; import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; @@ -96,7 +98,12 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { private final ForUtil forUtil; private final Lucene50SkipWriter skipWriter; - + + private boolean fieldHasNorms; + private NumericDocValues norms; + private CompetitiveFreqNormAccumulator competitiveFreqNormAccumulator; + private int maxFreq; + /** Creates a postings writer */ public Lucene50PostingsWriter(SegmentWriteState state) throws IOException { final float acceptableOverheadRatio = PackedInts.COMPACT; @@ -183,8 +190,16 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { @Override public int setField(FieldInfo fieldInfo) { super.setField(fieldInfo); - skipWriter.setField(writePositions, writeOffsets, writePayloads); + skipWriter.setField(writePositions, writeOffsets, writePayloads, fieldInfo.hasNorms()); lastState = emptyState; + fieldHasNorms = fieldInfo.hasNorms(); + if (fieldHasNorms) { + competitiveFreqNormAccumulator = new CompetitiveFreqNormAccumulator(); + maxFreq = -1; + } else { + competitiveFreqNormAccumulator = null; + maxFreq = 0; + } if (writePositions) { if (writePayloads || writeOffsets) { return 3; // doc + pos + pay FP @@ -197,7 +212,7 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { } @Override - public void startTerm() { + public void startTerm(NumericDocValues norms) { docStartFP = docOut.getFilePointer(); if (writePositions) { posStartFP = posOut.getFilePointer(); @@ -208,6 +223,17 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { lastDocID = 0; lastBlockDocID = -1; skipWriter.resetSkip(); + this.norms = norms; + if (fieldHasNorms) { + assert norms != null; + assert competitiveFreqNormAccumulator != null; + assert maxFreq == -1; + competitiveFreqNormAccumulator.clear(); + } else { + assert norms == null; + assert competitiveFreqNormAccumulator == null; + maxFreq = 0; + } } @Override @@ -216,7 +242,13 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { // Should write skip data as well as postings list for // current block. if (lastBlockDocID != -1 && docBufferUpto == 0) { - skipWriter.bufferSkip(lastBlockDocID, docCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockPayloadByteUpto); + skipWriter.bufferSkip(lastBlockDocID, competitiveFreqNormAccumulator, maxFreq, docCount, + lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockPayloadByteUpto); + if (fieldHasNorms) { + competitiveFreqNormAccumulator.clear(); + } else { + maxFreq = 0; + } } final int docDelta = docID - lastDocID; @@ -247,6 +279,14 @@ public final class Lucene50PostingsWriter extends PushPostingsWriterBase { lastDocID = docID; lastPosition = 0; lastStartOffset = 0; + + if (fieldHasNorms) { + boolean found = norms.advanceExact(docID); + assert found; + competitiveFreqNormAccumulator.add(termDocFreq, norms.longValue()); + } else { + maxFreq = Math.max(maxFreq, termDocFreq); + } } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java index 8c037c5..d9a3361 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipReader.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import org.apache.lucene.codecs.MultiLevelSkipListReader; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZE; @@ -53,11 +54,13 @@ import static org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.BLOCK_SIZ * */ final class Lucene50SkipReader extends MultiLevelSkipListReader { - private long docPointer[]; - private long posPointer[]; - private long payPointer[]; - private int posBufferUpto[]; - private int payloadByteUpto[]; + private final boolean fieldHasNorms; + private final long docPointer[]; + private final long posPointer[]; + private final long payPointer[]; + private final int posBufferUpto[]; + private final int payloadByteUpto[]; + private final float[] maxScore; private long lastPosPointer; private long lastPayPointer; @@ -65,7 +68,9 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { private long lastDocPointer; private int lastPosBufferUpto; - public Lucene50SkipReader(IndexInput skipStream, int maxSkipLevels, boolean hasPos, boolean hasOffsets, boolean hasPayloads) { + private SimScorer scorer; + + public Lucene50SkipReader(IndexInput skipStream, int maxSkipLevels, boolean hasPos, boolean hasOffsets, boolean hasPayloads, boolean hasNorms, SimScorer scorer) { super(skipStream, maxSkipLevels, BLOCK_SIZE, 8); docPointer = new long[maxSkipLevels]; if (hasPos) { @@ -83,7 +88,17 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { } } else { posPointer = null; + posBufferUpto = null; + payloadByteUpto = null; + payPointer = null; } + this.maxScore = new float[maxSkipLevels]; + this.fieldHasNorms = hasNorms; + this.scorer = scorer; + } + + void setScorer(SimScorer scorer) { + this.scorer = scorer; } /** @@ -114,6 +129,7 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { } else { assert posBasePointer == 0; } + Arrays.fill(maxScore, Float.POSITIVE_INFINITY); } /** Returns the doc pointer of the doc to which the last call of @@ -142,6 +158,18 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { return skipDoc[0]; } + public int skipToScore(float minScore) throws IOException { + int level = -1; + while (level < numberOfSkipLevels - 1 && maxScore[level + 1] < minScore) { + level++; + } + if (level == -1) { + return -1; + } else { + return skipDoc[level]; + } + } + @Override protected void seekChild(int level) throws IOException { super.seekChild(level); @@ -192,6 +220,44 @@ final class Lucene50SkipReader extends MultiLevelSkipListReader { payPointer[level] += skipStream.readVLong(); } } + + //StringBuilder sb = new StringBuilder(skipDoc[level] + "-" + (skipDoc[level] + delta) + ": "); + float maxScore = 0; + if (fieldHasNorms) { + if (level > 0) { + long length = skipStream.readVLong(); + long upTo = skipStream.getFilePointer() + length; + if (scorer != null) { + int freq = 0; + long norm = 0; + while (skipStream.getFilePointer() < upTo) { + freq += skipStream.readVInt(); + norm += skipStream.readZLong(); + //sb.append("(").append(freq).append(",").append(norm).append("),"); + maxScore = Math.max(maxScore, scorer.score(freq, Long.valueOf(norm))); + } + assert skipStream.getFilePointer() == upTo; + } else { + skipStream.seek(upTo); + maxScore = Float.POSITIVE_INFINITY; + } + } else if (numberOfSkipLevels > 1) { + // use the max score that was computed on level 1 instead + maxScore = this.maxScore[1]; + } + } else { + long maxFreq = skipStream.readVInt(); + if (scorer != null) { + maxScore = scorer.score(maxFreq, null); + } else { + maxScore = Float.POSITIVE_INFINITY; + } + } + this.maxScore[level] = maxScore; + if (level > 1) { + //System.out.println("max score level " + level + ": [" + skipDoc[level] + ":" + (skipDoc[level] + delta) + "]: " + maxScore); + //System.out.println(sb); + } return delta; } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java index a4556c6..414a6eb 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50SkipWriter.java @@ -19,9 +19,13 @@ package org.apache.lucene.codecs.lucene50; import java.io.IOException; import java.util.Arrays; +import java.util.Set; -import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator; +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm; import org.apache.lucene.codecs.MultiLevelSkipListWriter; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; /** * Write skip lists with multiple levels, and support skip within block ints. @@ -44,6 +48,7 @@ import org.apache.lucene.codecs.MultiLevelSkipListWriter; * */ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { + private final int maxSkipLevels; private int[] lastSkipDoc; private long[] lastSkipDocPointer; private long[] lastSkipPosPointer; @@ -60,12 +65,16 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { private long curPayPointer; private int curPosBufferUpto; private int curPayloadByteUpto; + private int[] curMaxFreq; + private CompetitiveFreqNormAccumulator[] curCompetitiveFreqNorms; private boolean fieldHasPositions; private boolean fieldHasOffsets; private boolean fieldHasPayloads; + private boolean fieldHasNorms; public Lucene50SkipWriter(int maxSkipLevels, int blockSize, int docCount, IndexOutput docOut, IndexOutput posOut, IndexOutput payOut) { super(blockSize, 8, maxSkipLevels, docCount); + this.maxSkipLevels = maxSkipLevels; this.docOut = docOut; this.posOut = posOut; this.payOut = payOut; @@ -81,10 +90,23 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { } } - public void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) { + public void setField(boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads, + boolean fieldHasNorms) { this.fieldHasPositions = fieldHasPositions; this.fieldHasOffsets = fieldHasOffsets; this.fieldHasPayloads = fieldHasPayloads; + this.fieldHasNorms = fieldHasNorms; + if (fieldHasNorms) { + curMaxFreq = null; + curCompetitiveFreqNorms = new CompetitiveFreqNormAccumulator[maxSkipLevels]; + // we do not store competitive (freq,norm) pairs on the first level + for (int i = 1; i < maxSkipLevels; ++i) { + curCompetitiveFreqNorms[i] = new CompetitiveFreqNormAccumulator(); + } + } else { + curMaxFreq = new int[maxSkipLevels]; + curCompetitiveFreqNorms = null; + } } // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the skipper @@ -122,6 +144,13 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { Arrays.fill(lastSkipPayPointer, lastPayFP); } } + if (fieldHasNorms) { + for (int i = 1; i < curCompetitiveFreqNorms.length; ++i) { + curCompetitiveFreqNorms[i].clear(); + } + } else { + Arrays.fill(curMaxFreq, 0); + } initialized = true; } } @@ -129,7 +158,8 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { /** * Sets the values for the current skip data. */ - public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException { + public void bufferSkip(int doc, CompetitiveFreqNormAccumulator competitiveFreqNorms, int maxFreq, + int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException { initSkip(); this.curDoc = doc; this.curDocPointer = docOut.getFilePointer(); @@ -137,11 +167,21 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { this.curPayPointer = payFP; this.curPosBufferUpto = posBufferUpto; this.curPayloadByteUpto = payloadByteUpto; + if (fieldHasNorms) { + assert maxFreq == -1; + this.curCompetitiveFreqNorms[1].addAll(competitiveFreqNorms); + } else { + assert competitiveFreqNorms == null; + this.curMaxFreq[0] = Math.max(maxFreq, curMaxFreq[0]); + } bufferSkip(numDocs); } - + + private final RAMOutputStream freqNormOut = new RAMOutputStream(); + @Override protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + int delta = curDoc - lastSkipDoc[level]; skipBuffer.writeVInt(delta); @@ -165,5 +205,31 @@ final class Lucene50SkipWriter extends MultiLevelSkipListWriter { lastSkipPayPointer[level] = curPayPointer; } } + + if (fieldHasNorms) { + if (level > 0) { // we don't write competitive (freq,norm) pairs on level 0 + CompetitiveFreqNormAccumulator competitiveFreqNorms = curCompetitiveFreqNorms[level]; + if (level + 1 < numberOfSkipLevels) { + curCompetitiveFreqNorms[level + 1].addAll(competitiveFreqNorms); + } + Set freqAndNorms = competitiveFreqNorms.getCompetitiveFreqNormPairs(); + FreqAndNorm previous = new FreqAndNorm(0, 0); + for (FreqAndNorm freqAndNorm : freqAndNorms) { + freqNormOut.writeVInt(freqAndNorm.freq - previous.freq); + freqNormOut.writeZLong(freqAndNorm.norm - previous.norm); + previous = freqAndNorm; + } + skipBuffer.writeVLong(freqNormOut.getFilePointer()); + freqNormOut.writeTo(skipBuffer); + freqNormOut.reset(); + competitiveFreqNorms.clear(); + } + } else { + if (level + 1 < numberOfSkipLevels) { + curMaxFreq[level + 1] = Math.max(curMaxFreq[level + 1], curMaxFreq[level]); + } + skipBuffer.writeVInt(curMaxFreq[level]); + curMaxFreq[level] = 0; + } } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java index 386655e..e758b13 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene70/Lucene70DocValuesProducer.java @@ -37,6 +37,7 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.RandomAccessInput; @@ -1153,7 +1154,7 @@ final class Lucene70DocValuesProducer extends DocValuesProducer implements Close } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java index 281b08f..36f0358 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java @@ -34,6 +34,7 @@ import java.util.TreeSet; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -117,7 +118,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { Map formatToGroups = buildFieldsGroupMapping(fields); // Write postings @@ -137,7 +138,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { FieldsConsumer consumer = format.fieldsConsumer(group.state); toClose.add(consumer); - consumer.write(maskedFields); + consumer.write(maskedFields, norms); } success = true; } finally { @@ -148,7 +149,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { } @Override - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { Map formatToGroups = buildFieldsGroupMapping(new MultiFields(mergeState.fieldsProducers, null)); // Merge postings @@ -161,7 +162,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { FieldsConsumer consumer = format.fieldsConsumer(group.state); toClose.add(consumer); - consumer.merge(pfMergeState.apply(group.fields)); + consumer.merge(pfMergeState.apply(group.fields), norms); } success = true; } finally { diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java index c676568..6e7c375 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java @@ -1083,7 +1083,7 @@ public final class CheckIndex implements Closeable { } } - postingsEnum = termsEnum.postings(postingsEnum, 0); + postingsEnum = termsEnum.postings(postingsEnum, null, 0); int lastDoc = -1; while (true) { @@ -1370,7 +1370,7 @@ public final class CheckIndex implements Closeable { } sumDocFreq += docFreq; - postings = termsEnum.postings(postings, PostingsEnum.ALL); + postings = termsEnum.postings(postings, null, PostingsEnum.ALL); if (hasFreqs == false) { if (termsEnum.totalTermFreq() != termsEnum.docFreq()) { @@ -1512,7 +1512,7 @@ public final class CheckIndex implements Closeable { if (hasPositions) { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - postings = termsEnum.postings(postings, PostingsEnum.ALL); + postings = termsEnum.postings(postings, null, PostingsEnum.ALL); final int docID = postings.advance(skipDocID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; @@ -1576,7 +1576,7 @@ public final class CheckIndex implements Closeable { } else { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - postings = termsEnum.postings(postings, PostingsEnum.NONE); + postings = termsEnum.postings(postings, null, PostingsEnum.NONE); final int docID = postings.advance(skipDocID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; @@ -1658,7 +1658,7 @@ public final class CheckIndex implements Closeable { } int expectedDocFreq = termsEnum.docFreq(); - PostingsEnum d = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum d = termsEnum.postings(null, null, PostingsEnum.NONE); int docFreq = 0; while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { docFreq++; @@ -1705,7 +1705,7 @@ public final class CheckIndex implements Closeable { throw new RuntimeException("seek to existing term " + seekTerms[i] + " returned FOUND but seeked to the wrong term " + termsEnum.term()); } - postings = termsEnum.postings(postings, PostingsEnum.NONE); + postings = termsEnum.postings(postings, null, PostingsEnum.NONE); if (postings == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } @@ -2463,7 +2463,7 @@ public final class CheckIndex implements Closeable { while ((term = termsEnum.next()) != null) { // This is the term vectors: - postings = termsEnum.postings(postings, PostingsEnum.ALL); + postings = termsEnum.postings(postings, null, PostingsEnum.ALL); assert postings != null; if (!postingsTermsEnum.seekExact(term)) { @@ -2471,7 +2471,7 @@ public final class CheckIndex implements Closeable { } // This is the inverted index ("real" postings): - postingsDocs = postingsTermsEnum.postings(postingsDocs, PostingsEnum.ALL); + postingsDocs = postingsTermsEnum.postings(postingsDocs, null, PostingsEnum.ALL); assert postingsDocs != null; diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index fd24105..b16c52c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.Closeable; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -31,6 +32,7 @@ import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.NormsConsumer; import org.apache.lucene.codecs.NormsFormat; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsFormat; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.document.FieldType; @@ -126,6 +128,7 @@ final class DefaultIndexingChain extends DocConsumer { if (docState.infoStream.isEnabled("IW")) { docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write norms"); } + SegmentReadState readState = new SegmentReadState(state.directory, state.segmentInfo, state.fieldInfos, IOContext.READ, state.segmentSuffix); t0 = System.nanoTime(); writeDocValues(state, sortMap); @@ -158,8 +161,32 @@ final class DefaultIndexingChain extends DocConsumer { perField = perField.next; } } - - termsHash.flush(fieldsToFlush, state, sortMap); + final NormsProducer norms; + if (readState.fieldInfos.hasNorms()) { + norms = state.segmentInfo.getCodec().normsFormat().normsProducer(readState); + } else { + norms = new NormsProducer() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() throws IOException {} + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void checkIntegrity() throws IOException {} + }; + } + try (Closeable c = norms) { + termsHash.flush(fieldsToFlush, state, sortMap, norms); + } if (docState.infoStream.isEnabled("IW")) { docState.infoStream.message("IW", ((System.nanoTime()-t0)/1000000) + " msec to write postings and finish vectors"); } diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java index 0450038..c02a9f6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Iterator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -210,8 +211,8 @@ public abstract class FilterLeafReader extends LeafReader { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - return in.postings(reuse, flags); + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { + return in.postings(reuse, scorer, flags); } } diff --git a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java index 6498dc0..9f14f89 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; /** @@ -178,8 +179,8 @@ public abstract class FilteredTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - return tenum.postings(reuse, flags); + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { + return tenum.postings(reuse, scorer, flags); } /** This enum does not support seeking! diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java index fb78a92..e23c016 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -225,7 +226,7 @@ class FreqProxFields extends Fields { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { FreqProxPostingsEnum posEnum; diff --git a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index d953f8d..aeef62d 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.util.CollectionUtil; import org.apache.lucene.util.IOUtils; @@ -55,7 +56,7 @@ final class FreqProxTermsWriter extends TermsHash { } if (termsEnum != null && termsEnum.seekExact(deleteTerm.bytes())) { - postingsEnum = termsEnum.postings(postingsEnum, 0); + postingsEnum = termsEnum.postings(postingsEnum, null, 0); int delDocLimit = segDeletes.get(deleteTerm); assert delDocLimit < PostingsEnum.NO_MORE_DOCS; while (true) { @@ -78,8 +79,9 @@ final class FreqProxTermsWriter extends TermsHash { } @Override - public void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { - super.flush(fieldsToFlush, state, sortMap); + public void flush(Map fieldsToFlush, final SegmentWriteState state, + Sorter.DocMap sortMap, NormsProducer norms) throws IOException { + super.flush(fieldsToFlush, state, sortMap, norms); // Gather all fields that saw any postings: List allFields = new ArrayList<>(); @@ -105,7 +107,7 @@ final class FreqProxTermsWriter extends TermsHash { FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state); boolean success = false; try { - consumer.write(fields); + consumer.write(fields, norms); success = true; } finally { if (success) { diff --git a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java index 202bf2c..be515e0 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java +++ b/lucene/core/src/java/org/apache/lucene/index/FrozenBufferedUpdates.java @@ -592,7 +592,7 @@ class FrozenBufferedUpdates { if (termsEnum.seekExact(term)) { // we don't need term frequencies for this - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.NONE); DocValuesFieldUpdates dvUpdates = holder.get(updateField); if (dvUpdates == null) { @@ -784,7 +784,7 @@ class FrozenBufferedUpdates { } // we don't need term frequencies for this - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.NONE); assert postingsEnum != null; diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java index faea32d..c7e74e8 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java @@ -147,7 +147,7 @@ public abstract class LeafReader extends IndexReader { if (terms != null) { final TermsEnum termsEnum = terms.iterator(); if (termsEnum.seekExact(term.bytes())) { - return termsEnum.postings(null, flags); + return termsEnum.postings(null, null, flags); } } return null; diff --git a/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java index 280b52b..661db19 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java @@ -19,6 +19,8 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.search.similarities.Similarity.SimScorer; + import static org.apache.lucene.index.FilterLeafReader.FilterFields; import static org.apache.lucene.index.FilterLeafReader.FilterTerms; import static org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; @@ -111,7 +113,7 @@ public class MappedMultiFields extends FilterFields { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { MappingMultiPostingsEnum mappingDocsAndPositionsEnum; if (reuse instanceof MappingMultiPostingsEnum) { MappingMultiPostingsEnum postings = (MappingMultiPostingsEnum) reuse; @@ -124,7 +126,7 @@ public class MappedMultiFields extends FilterFields { mappingDocsAndPositionsEnum = new MappingMultiPostingsEnum(field, mergeState); } - MultiPostingsEnum docsAndPositionsEnum = (MultiPostingsEnum) in.postings(mappingDocsAndPositionsEnum.multiDocsAndPositionsEnum, flags); + MultiPostingsEnum docsAndPositionsEnum = (MultiPostingsEnum) in.postings(mappingDocsAndPositionsEnum.multiDocsAndPositionsEnum, scorer, flags); mappingDocsAndPositionsEnum.reset(docsAndPositionsEnum); return mappingDocsAndPositionsEnum; } diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java index 1a7b15b..fa7b8ab 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiFields.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiFields.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.MergedIterator; @@ -155,7 +156,7 @@ public final class MultiFields extends Fields { * Some codecs may be able to optimize their * implementation when freqs are not required. This will * return null if the field or term does not exist. See {@link - * TermsEnum#postings(PostingsEnum,int)}.*/ + * TermsEnum#postings(PostingsEnum,SimScorer,int)}.*/ public static PostingsEnum getTermDocsEnum(IndexReader r, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; @@ -163,7 +164,7 @@ public final class MultiFields extends Fields { if (terms != null) { final TermsEnum termsEnum = terms.iterator(); if (termsEnum.seekExact(term)) { - return termsEnum.postings(null, flags); + return termsEnum.postings(null, null, flags); } } return null; @@ -182,7 +183,7 @@ public final class MultiFields extends Fields { * required. Some codecs may be able to optimize * their implementation when offsets and/or payloads are not * required. This will return null if the field or term does not - * exist. See {@link TermsEnum#postings(PostingsEnum,int)}. */ + * exist. See {@link TermsEnum#postings(PostingsEnum,SimScorer,int)}. */ public static PostingsEnum getTermPositionsEnum(IndexReader r, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; @@ -190,7 +191,7 @@ public final class MultiFields extends Fields { if (terms != null) { final TermsEnum termsEnum = terms.iterator(); if (termsEnum.seekExact(term)) { - return termsEnum.postings(null, flags); + return termsEnum.postings(null, null, flags); } } return null; diff --git a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java index 7db838b..e656454 100644 --- a/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Comparator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -333,7 +334,7 @@ public final class MultiTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { MultiPostingsEnum docsEnum; // Can only reuse if incoming enum is also a MultiDocsEnum @@ -356,7 +357,7 @@ public final class MultiTermsEnum extends TermsEnum { final TermsEnumWithSlice entry = top[i]; assert entry.index < docsEnum.subPostingsEnums.length: entry.index + " vs " + docsEnum.subPostingsEnums.length + "; " + subs.length; - final PostingsEnum subPostingsEnum = entry.terms.postings(docsEnum.subPostingsEnums[entry.index], flags); + final PostingsEnum subPostingsEnum = entry.terms.postings(docsEnum.subPostingsEnums[entry.index], scorer, flags); assert subPostingsEnum != null; docsEnum.subPostingsEnums[entry.index] = subPostingsEnum; subDocs[upto].postingsEnum = subPostingsEnum; diff --git a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java index fdd32a9..fc38308 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/PostingsEnum.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -29,29 +30,29 @@ import org.apache.lucene.util.BytesRef; public abstract class PostingsEnum extends DocIdSetIterator { /** - * Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} if you don't + * Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} if you don't * require per-document postings in the returned enum. */ public static final short NONE = 0; - /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} + /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} * if you require term frequencies in the returned enum. */ public static final short FREQS = 1 << 3; - /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} + /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} * if you require term positions in the returned enum. */ public static final short POSITIONS = FREQS | 1 << 4; - /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} + /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} * if you require offsets in the returned enum. */ public static final short OFFSETS = POSITIONS | 1 << 5; - /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} + /** Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} * if you require payloads in the returned enum. */ public static final short PAYLOADS = POSITIONS | 1 << 6; /** - * Flag to pass to {@link TermsEnum#postings(PostingsEnum, int)} + * Flag to pass to {@link TermsEnum#postings(PostingsEnum, SimScorer, int)} * to get positions, payloads and offsets in the returned enum */ public static final short ALL = OFFSETS | PAYLOADS; @@ -81,7 +82,15 @@ public abstract class PostingsEnum extends DocIdSetIterator { * the result of this method is undefined. */ public abstract int freq() throws IOException; - + + /** + * Like {@link #advance(int)} but may skip over documents whose score is + * less than {@code score}. + */ + public int advance(int target, float score) throws IOException { + return advance(target); + } + /** Returns the related attributes. */ public AttributeSource attributes() { if (atts == null) atts = new AttributeSource(); diff --git a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java index c67b92d..7baa1c7 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/core/src/java/org/apache/lucene/index/SegmentMerger.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.Closeable; import java.io.IOException; import java.util.List; @@ -24,6 +25,7 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.NormsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PointsWriter; import org.apache.lucene.codecs.StoredFieldsWriter; import org.apache.lucene.codecs.TermVectorsWriter; @@ -109,10 +111,47 @@ final class SegmentMerger { final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, null, context); + final SegmentReadState segmentReadState = new SegmentReadState(directory, mergeState.segmentInfo, mergeState.mergeFieldInfos, + IOContext.READ, segmentWriteState.segmentSuffix); + + final NormsProducer normsProducer; + if (mergeState.mergeFieldInfos.hasNorms()) { + if (mergeState.infoStream.isEnabled("SM")) { + t0 = System.nanoTime(); + } + mergeNorms(segmentWriteState); + if (mergeState.infoStream.isEnabled("SM")) { + long t1 = System.nanoTime(); + mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge norms [" + numMerged + " docs]"); + } + normsProducer = codec.normsFormat().normsProducer(segmentReadState); + } else { + normsProducer = new NormsProducer() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() throws IOException {} + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void checkIntegrity() throws IOException {} + }; + } + if (mergeState.infoStream.isEnabled("SM")) { t0 = System.nanoTime(); } - mergeTerms(segmentWriteState); + try (Closeable c = normsProducer) { + mergeTerms(segmentWriteState, normsProducer); + } if (mergeState.infoStream.isEnabled("SM")) { long t1 = System.nanoTime(); mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge postings [" + numMerged + " docs]"); @@ -139,17 +178,6 @@ final class SegmentMerger { long t1 = System.nanoTime(); mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge points [" + numMerged + " docs]"); } - - if (mergeState.mergeFieldInfos.hasNorms()) { - if (mergeState.infoStream.isEnabled("SM")) { - t0 = System.nanoTime(); - } - mergeNorms(segmentWriteState); - if (mergeState.infoStream.isEnabled("SM")) { - long t1 = System.nanoTime(); - mergeState.infoStream.message("SM", ((t1-t0)/1000000) + " msec to merge norms [" + numMerged + " docs]"); - } - } if (mergeState.mergeFieldInfos.hasVectors()) { if (mergeState.infoStream.isEnabled("SM")) { @@ -225,9 +253,9 @@ final class SegmentMerger { } } - private void mergeTerms(SegmentWriteState segmentWriteState) throws IOException { + private void mergeTerms(SegmentWriteState segmentWriteState, NormsProducer norms) throws IOException { try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) { - consumer.merge(mergeState); + consumer.merge(mergeState, norms); } } } diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java index ccee7a3..3598c5e 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -105,7 +106,7 @@ class SortedDocValuesTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java index eba95c9..6d80b16 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -105,7 +106,7 @@ class SortedSetDocValuesTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { throw new UnsupportedOperationException(); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java index eb9f7ed..0672905 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortingLeafReader.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMFile; @@ -111,7 +112,7 @@ class SortingLeafReader extends FilterLeafReader { } @Override - public PostingsEnum postings( PostingsEnum reuse, final int flags) throws IOException { + public PostingsEnum postings( PostingsEnum reuse, SimScorer scorer, final int flags) throws IOException { if (hasPositions && PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { final PostingsEnum inReuse; @@ -126,7 +127,7 @@ class SortingLeafReader extends FilterLeafReader { inReuse = reuse; } - final PostingsEnum inDocsAndPositions = in.postings(inReuse, flags); + final PostingsEnum inDocsAndPositions = in.postings(inReuse, scorer, flags); // we ignore the fact that offsets may be stored but not asked for, // since this code is expected to be used during addIndexes which will // ask for everything. if that assumption changes in the future, we can @@ -147,7 +148,7 @@ class SortingLeafReader extends FilterLeafReader { inReuse = reuse; } - final PostingsEnum inDocs = in.postings(inReuse, flags); + final PostingsEnum inDocs = in.postings(inReuse, scorer, flags); final boolean withFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >=0 && PostingsEnum.featureRequested(flags, PostingsEnum.FREQS); return new SortingDocsEnum(docMap.size(), wrapReuse, inDocs, withFreqs, docMap); } diff --git a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java index dff808e..9dee90f 100644 --- a/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/SortingTermVectorsConsumer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Iterator; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.codecs.TermVectorsWriter; import org.apache.lucene.search.DocIdSetIterator; @@ -37,8 +38,8 @@ final class SortingTermVectorsConsumer extends TermVectorsConsumer { } @Override - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { - super.flush(fieldsToFlush, state, sortMap); + void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap, NormsProducer norms) throws IOException { + super.flush(fieldsToFlush, state, sortMap, norms); if (tmpDirectory != null) { if (sortMap == null) { // we're lucky the index is already sorted, just rename the temporary file and return @@ -152,7 +153,7 @@ final class SortingTermVectorsConsumer extends TermVectorsConsumer { writer.startTerm(termsEnum.term(), freq); if (hasPositions || hasOffsets) { - docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS | PostingsEnum.PAYLOADS); assert docsAndPositionsEnum != null; final int docID = docsAndPositionsEnum.nextDoc(); diff --git a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java index 46dc63c..1ac20dd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.TermVectorsWriter; import org.apache.lucene.store.FlushInfo; import org.apache.lucene.store.IOContext; @@ -53,7 +54,7 @@ class TermVectorsConsumer extends TermsHash { } @Override - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { + void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap, NormsProducer norms) throws IOException { if (writer != null) { int numDocs = state.segmentInfo.maxDoc(); assert numDocs > 0; diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java index 4b5755a..596bfb7 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; @@ -146,13 +147,13 @@ public abstract class TermsEnum implements BytesRefIterator { * Use this method if you only require documents and frequencies, * and do not need any proximity data. * This method is equivalent to - * {@link #postings(PostingsEnum, int) postings(reuse, PostingsEnum.FREQS)} + * {@link #postings(PostingsEnum, SimScorer, int) postings(reuse, PostingsEnum.FREQS)} * * @param reuse pass a prior PostingsEnum for possible reuse - * @see #postings(PostingsEnum, int) + * @see #postings(PostingsEnum, SimScorer, int) */ public final PostingsEnum postings(PostingsEnum reuse) throws IOException { - return postings(reuse, PostingsEnum.FREQS); + return postings(reuse, null, PostingsEnum.FREQS); } /** Get {@link PostingsEnum} for the current term, with @@ -168,8 +169,13 @@ public abstract class TermsEnum implements BytesRefIterator { * @param flags specifies which optional per-document values * you require; see {@link PostingsEnum#FREQS} */ - public abstract PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException; + public abstract PostingsEnum postings(PostingsEnum reuse, SimScorer simScorer, int flags) throws IOException; + // nocommit + public final PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + return postings(reuse, null, flags); + } + /** * Expert: Returns the TermsEnums internal state to position the TermsEnum * without re-seeking the term dictionary. @@ -225,7 +231,7 @@ public abstract class TermsEnum implements BytesRefIterator { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { throw new IllegalStateException("this method should never be called"); } diff --git a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java index bede2f8..f420aca 100644 --- a/lucene/core/src/java/org/apache/lucene/index/TermsHash.java +++ b/lucene/core/src/java/org/apache/lucene/index/TermsHash.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.Counter; import org.apache.lucene.util.IntBlockPool; @@ -76,13 +77,14 @@ abstract class TermsHash { bytePool.reset(false, false); } - void flush(Map fieldsToFlush, final SegmentWriteState state, Sorter.DocMap sortMap) throws IOException { + void flush(Map fieldsToFlush, final SegmentWriteState state, + Sorter.DocMap sortMap, NormsProducer norms) throws IOException { if (nextTermsHash != null) { Map nextChildFields = new HashMap<>(); for (final Map.Entry entry : fieldsToFlush.entrySet()) { nextChildFields.put(entry.getKey(), entry.getValue().nextPerField); } - nextTermsHash.flush(nextChildFields, state, sortMap); + nextTermsHash.flush(nextChildFields, state, sortMap, norms); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java index 881c5dd..eb5313d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java +++ b/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.Attribute; import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.AttributeReflector; @@ -268,8 +269,8 @@ public final class FuzzyTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - return actualEnum.postings(reuse, flags); + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { + return actualEnum.postings(reuse, scorer, flags); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 34361a7..4cb367b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -256,7 +256,7 @@ public class MultiPhraseQuery extends Query { TermState termState = termContexts.get(term).get(context.ord); if (termState != null) { termsEnum.seekExact(term.bytes(), termState); - postings.add(termsEnum.postings(null, PostingsEnum.POSITIONS)); + postings.add(termsEnum.postings(null, null, PostingsEnum.POSITIONS)); totalMatchCost += PhraseQuery.termPositionsCost(termsEnum); } } diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java index f82316d..ee2482c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java +++ b/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryConstantScoreWrapper.java @@ -163,14 +163,14 @@ final class MultiTermQueryConstantScoreWrapper extends TermsEnum termsEnum2 = terms.iterator(); for (TermAndState t : collectedTerms) { termsEnum2.seekExact(t.term, t.state); - docs = termsEnum2.postings(docs, PostingsEnum.NONE); + docs = termsEnum2.postings(docs, null, PostingsEnum.NONE); builder.add(docs); } } // Then keep filling the bit set with remaining terms do { - docs = termsEnum.postings(docs, PostingsEnum.NONE); + docs = termsEnum.postings(docs, null, PostingsEnum.NONE); builder.add(docs); } while (termsEnum.next() != null); diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java index 3d359b4..111410b 100644 --- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java @@ -421,7 +421,7 @@ public class PhraseQuery extends Query { return null; } te.seekExact(t.bytes(), state); - PostingsEnum postingsEnum = te.postings(null, PostingsEnum.POSITIONS); + PostingsEnum postingsEnum = te.postings(null, null, PostingsEnum.POSITIONS); postingsFreqs[i] = new PostingsAndFreq(postingsEnum, positions[i], t); totalMatchCost += termPositionsCost(te); } diff --git a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java index ce9d6e0..ed27d83 100644 --- a/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/SynonymQuery.java @@ -211,12 +211,12 @@ public final class SynonymQuery extends Query { maxFreq += getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq()); - PostingsEnum postings = termsEnum.postings(null, PostingsEnum.FREQS); + PostingsEnum postings = termsEnum.postings(null, null, PostingsEnum.FREQS); // lazy init sim, in case no terms exist if (simScorer == null) { simScorer = similarity.simScorer(simWeight, context); } - subScorers.add(new TermScorer(this, postings, simScorer, Float.POSITIVE_INFINITY)); + subScorers.add(new TermScorer(this, postings, simScorer, ScoreMode.COMPLETE, Float.POSITIVE_INFINITY)); } } if (subScorers.isEmpty()) { diff --git a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java index 4049e10..ec25341 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermInSetQuery.java @@ -246,18 +246,18 @@ public class TermInSetQuery extends Query implements Accountable { assert field.equals(iterator.field()); if (termsEnum.seekExact(term)) { if (matchingTerms == null) { - docs = termsEnum.postings(docs, PostingsEnum.NONE); + docs = termsEnum.postings(docs, null, PostingsEnum.NONE); builder.add(docs); } else if (matchingTerms.size() < threshold) { matchingTerms.add(new TermAndState(field, termsEnum)); } else { assert matchingTerms.size() == threshold; builder = new DocIdSetBuilder(reader.maxDoc(), terms); - docs = termsEnum.postings(docs, PostingsEnum.NONE); + docs = termsEnum.postings(docs, null, PostingsEnum.NONE); builder.add(docs); for (TermAndState t : matchingTerms) { t.termsEnum.seekExact(t.term, t.state); - docs = t.termsEnum.postings(docs, PostingsEnum.NONE); + docs = t.termsEnum.postings(docs, null, PostingsEnum.NONE); builder.add(docs); } matchingTerms = null; diff --git a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java index 925fe93..b18fac9 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermQuery.java @@ -48,21 +48,21 @@ public class TermQuery extends Query { private final Similarity similarity; private final Similarity.SimWeight stats; private final TermContext termStates; - private final boolean needsScores; + private final ScoreMode scoreMode; - public TermWeight(IndexSearcher searcher, boolean needsScores, + public TermWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost, TermContext termStates) throws IOException { super(TermQuery.this); - if (needsScores && termStates == null) { + if (scoreMode.needsScores() && termStates == null) { throw new IllegalStateException("termStates are required when scores are needed"); } - this.needsScores = needsScores; + this.scoreMode = scoreMode; this.termStates = termStates; - this.similarity = searcher.getSimilarity(needsScores); + this.similarity = searcher.getSimilarity(scoreMode.needsScores()); final CollectionStatistics collectionStats; final TermStatistics termStats; - if (needsScores) { + if (scoreMode.needsScores()) { collectionStats = searcher.collectionStatistics(term.field()); termStats = searcher.termStatistics(term, termStates); } else { @@ -99,9 +99,12 @@ public class TermQuery extends Query { .getFieldInfos() .fieldInfo(getTerm().field()) .getIndexOptions(); - PostingsEnum docs = termsEnum.postings(null, needsScores ? PostingsEnum.FREQS : PostingsEnum.NONE); + SimScorer scorer = similarity.simScorer(stats, context); + PostingsEnum docs = termsEnum.postings(null, + scoreMode == ScoreMode.TOP_SCORES ? scorer : null, + scoreMode.needsScores() ? PostingsEnum.FREQS : PostingsEnum.NONE); assert docs != null; - return new TermScorer(this, docs, similarity.simScorer(stats, context), + return new TermScorer(this, docs, scorer, scoreMode, getMaxFreq(indexOptions, termsEnum.totalTermFreq(), termsEnum.docFreq())); } @@ -221,7 +224,7 @@ public class TermQuery extends Query { termState = this.perReaderTermState; } - return new TermWeight(searcher, scoreMode.needsScores(), boost, termState); + return new TermWeight(searcher, scoreMode, boost, termState); } /** Prints a user-readable version of this query. */ diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java index a4aeb04..884184d 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java @@ -27,7 +27,9 @@ import org.apache.lucene.search.similarities.Similarity; final class TermScorer extends Scorer { private final PostingsEnum postingsEnum; private final Similarity.SimScorer docScorer; + private final ScoreMode scoreMode; private final float maxFreq; + private float minCompetitiveScore; /** * Construct a TermScorer. @@ -42,11 +44,12 @@ final class TermScorer extends Scorer { * @param maxFreq * An upper bound of the term frequency of the searched term in any document. */ - TermScorer(Weight weight, PostingsEnum td, Similarity.SimScorer docScorer, float maxFreq) { + TermScorer(Weight weight, PostingsEnum td, Similarity.SimScorer docScorer, ScoreMode scoreMode, float maxFreq) { super(weight); this.docScorer = docScorer; this.postingsEnum = td; this.maxFreq = maxFreq; + this.scoreMode = scoreMode; } @Override @@ -60,7 +63,45 @@ final class TermScorer extends Scorer { @Override public DocIdSetIterator iterator() { - return postingsEnum; + if (scoreMode != ScoreMode.TOP_SCORES) { + return postingsEnum; + } + return new DocIdSetIterator() { + + @Override + public int nextDoc() throws IOException { + if (minCompetitiveScore == 0) { + return postingsEnum.nextDoc(); + } else { + return postingsEnum.advance(docID() + 1, minCompetitiveScore); + } + } + + @Override + public int advance(int target) throws IOException { + if (minCompetitiveScore == 0) { + return postingsEnum.advance(target); + } else { + return postingsEnum.advance(target, minCompetitiveScore); + } + } + + @Override + public int docID() { + return postingsEnum.docID(); + } + + @Override + public long cost() { + return postingsEnum.cost(); + } + }; + } + + @Override + public void setMinCompetitiveScore(float minScore) { + this.minCompetitiveScore = minScore; + //System.out.println(docID() + " | " + minScore); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java index dce156b..1b02d9e 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/BM25Similarity.java @@ -225,6 +225,17 @@ public class BM25Similarity extends Similarity { } @Override + public float score(float freq, Long normL) { + double norm; + if (normL == null) { + norm = k1; + } else { + norm = cache[((byte) normL.longValue()) & 0xFF]; + } + return weightValue * (float) (freq / (freq + norm)); + } + + @Override public float maxScore(float maxFreq) { // TODO: leverage maxFreq and the min norm from the cache return weightValue; diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java index 5f0bcd0..c6048f6 100644 --- a/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java +++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java @@ -158,6 +158,10 @@ public abstract class Similarity { */ public abstract float score(int doc, float freq) throws IOException; + public float score(float freq, Long norm) { + throw new UnsupportedOperationException(); + } + /** * Return the maximum score that this scorer may produce for freqs in {@code ]0, maxFreq]}. * {@code Float.POSITIVE_INFINITY} is a fine return value if scores are not bounded. diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java index 9eea3aa..024e6c3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java @@ -122,7 +122,7 @@ public class SpanTermQuery extends SpanQuery { final TermsEnum termsEnum = terms.iterator(); termsEnum.seekExact(term.bytes(), state); - final PostingsEnum postings = termsEnum.postings(null, requiredPostings.getRequiredPostings()); + final PostingsEnum postings = termsEnum.postings(null, null, requiredPostings.getRequiredPostings()); float positionsCost = termPositionsCost(termsEnum) * PHRASE_TO_SPAN_TERM_POSITIONS_COST; return new TermSpans(getSimScorer(context), postings, term, positionsCost); } diff --git a/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java new file mode 100644 index 0000000..1870a11 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/codecs/TestCompetitiveFreqNormAccumulator.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.codecs.CompetitiveFreqNormAccumulator.FreqAndNorm; +import org.apache.lucene.util.LuceneTestCase; + +public class TestCompetitiveFreqNormAccumulator extends LuceneTestCase { + + public void testBasics() { + CompetitiveFreqNormAccumulator acc = new CompetitiveFreqNormAccumulator(); + Set expected = new HashSet<>(); + + acc.add(3, 5); + expected.add(new FreqAndNorm(3, 5)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(6, 11); + expected.add(new FreqAndNorm(6, 11)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(10, 13); + expected.add(new FreqAndNorm(10, 13)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(1, 2); + expected.add(new FreqAndNorm(1, 2)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(7, 9); + expected.remove(new FreqAndNorm(6, 11)); + expected.add(new FreqAndNorm(7, 9)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + + acc.add(8, 2); + expected.clear(); + expected.add(new FreqAndNorm(10, 13)); + expected.add(new FreqAndNorm(8, 2)); + assertEquals(expected, acc.getCompetitiveFreqNormPairs()); + } + +} diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java index 3b99ee1..8fb5c48 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java @@ -291,35 +291,35 @@ public class TestBlockPostingsFormat3 extends LuceneTestCase { if (deep) { if (hasPositions) { // with payloads + off - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.ALL)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.ALL)); // with payloads only - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.PAYLOADS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.PAYLOADS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.PAYLOADS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.PAYLOADS)); + leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.PAYLOADS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.PAYLOADS)); // with offsets only - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.OFFSETS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.OFFSETS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.OFFSETS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.OFFSETS)); + leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.OFFSETS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.OFFSETS)); // with positions only - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.POSITIONS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.POSITIONS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.POSITIONS), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.POSITIONS)); + leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.POSITIONS), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.POSITIONS)); } // with freqs: @@ -327,8 +327,8 @@ public class TestBlockPostingsFormat3 extends LuceneTestCase { rightDocs = rightTermsEnum.postings(rightDocs)); // w/o freqs: - assertDocsEnum(leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE)); + assertDocsEnum(leftDocs = leftTermsEnum.postings(leftDocs, null, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, null, PostingsEnum.NONE)); // with freqs: assertDocsSkipping(leftTermsEnum.docFreq(), @@ -337,8 +337,8 @@ public class TestBlockPostingsFormat3 extends LuceneTestCase { // w/o freqs: assertDocsSkipping(leftTermsEnum.docFreq(), - leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE)); + leftDocs = leftTermsEnum.postings(leftDocs, null, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, null, PostingsEnum.NONE)); } } assertNull(rightTermsEnum.next()); diff --git a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java index 804f507..84544bc 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.asserting.AssertingCodec; import org.apache.lucene.codecs.blockterms.LuceneVarGapFixedInterval; @@ -407,17 +408,17 @@ public class TestPerFieldPostingsFormat2 extends LuceneTestCase { final FieldsConsumer consumer = delegate.fieldsConsumer(state); return new FieldsConsumer() { @Override - public void write(Fields fields) throws IOException { - consumer.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + consumer.write(fields, norms); } @Override - public void merge(MergeState mergeState) throws IOException { + public void merge(MergeState mergeState, NormsProducer norms) throws IOException { nbMergeCalls++; for (FieldInfo fi : mergeState.mergeFieldInfos) { fieldNames.add(fi.name); } - consumer.merge(mergeState); + consumer.merge(mergeState, norms); } @Override diff --git a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java index 4625f73..abe29e1 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java @@ -17,6 +17,7 @@ package org.apache.lucene.index; +import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -28,10 +29,12 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.StringField; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -379,7 +382,7 @@ public class TestCodecs extends LuceneTestCase { if (field.omitTF) { this.verifyDocs(term.docs, term.positions, TestUtil.docs(random(), termsEnum, null, PostingsEnum.NONE), false); } else { - this.verifyDocs(term.docs, term.positions, termsEnum.postings(null, PostingsEnum.ALL), true); + this.verifyDocs(term.docs, term.positions, termsEnum.postings(null, null, PostingsEnum.ALL), true); } // Test random seek by ord: @@ -399,7 +402,7 @@ public class TestCodecs extends LuceneTestCase { if (field.omitTF) { this.verifyDocs(term.docs, term.positions, TestUtil.docs(random(), termsEnum, null, PostingsEnum.NONE), false); } else { - this.verifyDocs(term.docs, term.positions, termsEnum.postings(null, PostingsEnum.ALL), true); + this.verifyDocs(term.docs, term.positions, termsEnum.postings(null, null, PostingsEnum.ALL), true); } } @@ -451,7 +454,7 @@ public class TestCodecs extends LuceneTestCase { if (!field.omitTF) { // TODO: we should randomize which postings features are available, but // need to coordinate this with the checks below that rely on such features - postings = termsEnum.postings(null, PostingsEnum.ALL); + postings = termsEnum.postings(null, null, PostingsEnum.ALL); } else { postings = TestUtil.docs(random(), termsEnum, null, PostingsEnum.FREQS); } @@ -672,7 +675,7 @@ public class TestCodecs extends LuceneTestCase { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { return new DataPostingsEnum(fieldData.terms[upto]); } @@ -752,9 +755,27 @@ public class TestCodecs extends LuceneTestCase { Arrays.sort(fields); FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(state); + NormsProducer fakeNorms = new NormsProducer() { + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() throws IOException {} + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + return null; + } + + @Override + public void checkIntegrity() throws IOException {} + }; boolean success = false; try { - consumer.write(new DataFields(fields)); + consumer.write(new DataFields(fields), fakeNorms); success = true; } finally { if (success) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java index 7afcf7a..510ec2a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java @@ -636,8 +636,8 @@ public class TestDirectoryReader extends LuceneTestCase { while(enum1.next() != null) { assertEquals("Different terms", enum1.term(), enum2.next()); - PostingsEnum tp1 = enum1.postings(null, PostingsEnum.ALL); - PostingsEnum tp2 = enum2.postings(null, PostingsEnum.ALL); + PostingsEnum tp1 = enum1.postings(null, null, PostingsEnum.ALL); + PostingsEnum tp2 = enum2.postings(null, null, PostingsEnum.ALL); while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java index 37761d3..8a6a1ae 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDoc.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDoc.java @@ -261,7 +261,7 @@ public class TestDoc extends LuceneTestCase { out.print(" term=" + fieldInfo.name + ":" + tis.term()); out.println(" DF=" + tis.docFreq()); - PostingsEnum positions = tis.postings(null, PostingsEnum.POSITIONS); + PostingsEnum positions = tis.postings(null, null, PostingsEnum.POSITIONS); final Bits liveDocs = reader.getLiveDocs(); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java index cc1f2be..5291d4f 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java @@ -95,7 +95,7 @@ public class TestDocsAndPositions extends LuceneTestCase { if (terms != null) { TermsEnum te = terms.iterator(); if (te.seekExact(bytes)) { - return te.postings(null, PostingsEnum.ALL); + return te.postings(null, null, PostingsEnum.ALL); } } return null; @@ -368,7 +368,7 @@ public class TestDocsAndPositions extends LuceneTestCase { // now reuse and check again TermsEnum te = r.terms("foo").iterator(); assertTrue(te.seekExact(new BytesRef("bar"))); - disi = te.postings(disi, PostingsEnum.ALL); + disi = te.postings(disi, null, PostingsEnum.ALL); docid = disi.docID(); assertEquals(-1, docid); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java index de06f90..36ac659 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -63,8 +64,8 @@ public class TestFilterLeafReader extends LuceneTestCase { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { - return new TestPositions(super.postings(reuse == null ? null : ((FilterPostingsEnum) reuse).in, flags)); + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { + return new TestPositions(super.postings(reuse == null ? null : ((FilterPostingsEnum) reuse).in, scorer, flags)); } } @@ -149,7 +150,7 @@ public class TestFilterLeafReader extends LuceneTestCase { assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one"))); - PostingsEnum positions = terms.postings(null, PostingsEnum.ALL); + PostingsEnum positions = terms.postings(null, null, PostingsEnum.ALL); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue((positions.docID() % 2) == 1); } diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java index bbedc20..46c5889 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -768,14 +768,14 @@ public class TestIndexWriter extends LuceneTestCase { Terms tpv = r.getTermVectors(0).terms("field"); TermsEnum termsEnum = tpv.iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(100, dpEnum.nextPosition()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java index 510a899..3c06e78 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java @@ -273,14 +273,14 @@ public class TestIndexableField extends LuceneTestCase { TermsEnum termsEnum = tfv.iterator(); assertEquals(new BytesRef(""+counter), termsEnum.next()); assertEquals(1, termsEnum.totalTermFreq()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(1, dpEnum.nextPosition()); assertEquals(new BytesRef("text"), termsEnum.next()); assertEquals(1, termsEnum.totalTermFreq()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(0, dpEnum.nextPosition()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java index 33f044d..bc05ed7 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloads.java @@ -483,7 +483,7 @@ public class TestPayloads extends LuceneTestCase { PostingsEnum tp = null; while (terms.next() != null) { String termText = terms.term().utf8ToString(); - tp = terms.postings(tp, PostingsEnum.PAYLOADS); + tp = terms.postings(tp, null, PostingsEnum.PAYLOADS); while(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { int freq = tp.freq(); for (int i = 0; i < freq; i++) { @@ -604,7 +604,7 @@ public class TestPayloads extends LuceneTestCase { DirectoryReader reader = writer.getReader(); TermsEnum te = MultiFields.getTerms(reader, "field").iterator(); assertTrue(te.seekExact(new BytesRef("withPayload"))); - PostingsEnum de = te.postings(null, PostingsEnum.PAYLOADS); + PostingsEnum de = te.postings(null, null, PostingsEnum.PAYLOADS); de.nextDoc(); de.nextPosition(); assertEquals(new BytesRef("test"), de.getPayload()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java b/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java index a90a5d2..6a83fee 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java @@ -72,7 +72,7 @@ public class TestPayloadsOnVectors extends LuceneTestCase { assert terms != null; TermsEnum termsEnum = terms.iterator(); assertTrue(termsEnum.seekExact(new BytesRef("withPayload"))); - PostingsEnum de = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum de = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(0, de.nextDoc()); assertEquals(0, de.nextPosition()); assertEquals(new BytesRef("test"), de.getPayload()); @@ -114,7 +114,7 @@ public class TestPayloadsOnVectors extends LuceneTestCase { assert terms != null; TermsEnum termsEnum = terms.iterator(); assertTrue(termsEnum.seekExact(new BytesRef("withPayload"))); - PostingsEnum de = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum de = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(0, de.nextDoc()); assertEquals(3, de.nextPosition()); assertEquals(new BytesRef("test"), de.getPayload()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java index 600ce76..e468193 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java @@ -317,7 +317,7 @@ public class TestPostingsOffsets extends LuceneTestCase { } // explicitly exclude offsets here - docsAndPositions = termsEnum.postings(docsAndPositions, PostingsEnum.ALL); + docsAndPositions = termsEnum.postings(docsAndPositions, null, PostingsEnum.ALL); assertNotNull(docsAndPositions); //System.out.println(" doc/freq/pos"); while((doc = docsAndPositions.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { @@ -332,7 +332,7 @@ public class TestPostingsOffsets extends LuceneTestCase { } } - docsAndPositionsAndOffsets = termsEnum.postings(docsAndPositions, PostingsEnum.ALL); + docsAndPositionsAndOffsets = termsEnum.postings(docsAndPositions, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsAndOffsets); //System.out.println(" doc/freq/pos/offs"); while((doc = docsAndPositionsAndOffsets.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java index d386f39..3c39cb0 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java @@ -404,7 +404,7 @@ public class TestStressIndexing2 extends LuceneTestCase { BytesRef term2; while((term2 = termsEnum3.next()) != null) { System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq()); - dpEnum = termsEnum3.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum3.postings(dpEnum, null, PostingsEnum.ALL); if (terms3.hasPositions()) { assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); final int freq = dpEnum.freq(); @@ -436,7 +436,7 @@ public class TestStressIndexing2 extends LuceneTestCase { BytesRef term2; while((term2 = termsEnum3.next()) != null) { System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq()); - dpEnum = termsEnum3.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum3.postings(dpEnum, null, PostingsEnum.ALL); if (dpEnum != null) { assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); final int freq = dpEnum.freq(); @@ -632,8 +632,8 @@ public class TestStressIndexing2 extends LuceneTestCase { assertEquals(termsEnum1.totalTermFreq(), termsEnum2.totalTermFreq()); - dpEnum1 = termsEnum1.postings(dpEnum1, PostingsEnum.ALL); - dpEnum2 = termsEnum2.postings(dpEnum2, PostingsEnum.ALL); + dpEnum1 = termsEnum1.postings(dpEnum1, null, PostingsEnum.ALL); + dpEnum2 = termsEnum2.postings(dpEnum2, null, PostingsEnum.ALL); if (terms1.hasPositions()) { assertTrue(terms2.hasPositions()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java index 76947dd..b71c336 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java @@ -257,7 +257,7 @@ public class TestTermVectorsReader extends LuceneTestCase { //System.out.println("Term: " + term); assertEquals(testTerms[i], term); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertNotNull(dpEnum); int doc = dpEnum.docID(); assertEquals(-1, doc); @@ -268,7 +268,7 @@ public class TestTermVectorsReader extends LuceneTestCase { } assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); doc = dpEnum.docID(); assertEquals(-1, doc); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -294,7 +294,7 @@ public class TestTermVectorsReader extends LuceneTestCase { //System.out.println("Term: " + term); assertEquals(testTerms[i], term); assertNotNull(termsEnum.postings(null)); - assertNotNull(termsEnum.postings(null, PostingsEnum.ALL)); + assertNotNull(termsEnum.postings(null, null, PostingsEnum.ALL)); } reader.close(); } @@ -313,7 +313,7 @@ public class TestTermVectorsReader extends LuceneTestCase { String term = text.utf8ToString(); assertEquals(testTerms[i], term); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(dpEnum.freq(), positions[i].length); @@ -322,7 +322,7 @@ public class TestTermVectorsReader extends LuceneTestCase { } assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertNotNull(dpEnum); assertEquals(dpEnum.freq(), positions[i].length); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java index 11f1503..8431d51 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java @@ -68,7 +68,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { // Token "" occurred once assertEquals(1, termsEnum.totalTermFreq()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(8, dpEnum.startOffset()); @@ -77,7 +77,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { // Token "abcd" occurred three times assertEquals(new BytesRef("abcd"), termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertEquals(3, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -117,7 +117,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -152,7 +152,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -190,7 +190,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -225,7 +225,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -261,7 +261,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); @@ -269,14 +269,14 @@ public class TestTermVectorsWriter extends LuceneTestCase { assertEquals(4, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(11, dpEnum.startOffset()); assertEquals(17, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(18, dpEnum.startOffset()); @@ -305,7 +305,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -314,7 +314,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { assertEquals(7, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(8, dpEnum.startOffset()); @@ -347,7 +347,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(); assertNotNull(termsEnum.next()); - PostingsEnum dpEnum = termsEnum.postings(null, PostingsEnum.ALL); + PostingsEnum dpEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -356,7 +356,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { assertEquals(4, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.ALL); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.ALL); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(6, dpEnum.startOffset()); diff --git a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java index fcc043c..c267986 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -753,25 +753,25 @@ public class TestTermsEnum extends LuceneTestCase { CompiledAutomaton ca = new CompiledAutomaton(automaton, false, false); TermsEnum te = terms.intersect(ca, null); assertEquals("aaa", te.next().utf8ToString()); - assertEquals(0, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(0, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("bbb", te.next().utf8ToString()); - assertEquals(1, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(1, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("ccc", te.next().utf8ToString()); - assertEquals(2, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(2, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertNull(te.next()); te = terms.intersect(ca, new BytesRef("abc")); assertEquals("bbb", te.next().utf8ToString()); - assertEquals(1, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(1, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("ccc", te.next().utf8ToString()); - assertEquals(2, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(2, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertNull(te.next()); te = terms.intersect(ca, new BytesRef("aaa")); assertEquals("bbb", te.next().utf8ToString()); - assertEquals(1, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(1, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("ccc", te.next().utf8ToString()); - assertEquals(2, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(2, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertNull(te.next()); r.close(); @@ -811,17 +811,17 @@ public class TestTermsEnum extends LuceneTestCase { // should seek to startTerm te = terms.intersect(ca, new BytesRef("aad")); assertEquals("abd", te.next().utf8ToString()); - assertEquals(1, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(1, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("acd", te.next().utf8ToString()); - assertEquals(2, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(2, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertEquals("bcd", te.next().utf8ToString()); - assertEquals(3, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(3, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertNull(te.next()); // should fail to find ceil label on second arc, rewind te = terms.intersect(ca, new BytesRef("add")); assertEquals("bcd", te.next().utf8ToString()); - assertEquals(3, te.postings(null, PostingsEnum.NONE).nextDoc()); + assertEquals(3, te.postings(null, null, PostingsEnum.NONE).nextDoc()); assertNull(te.next()); // should reach end @@ -865,12 +865,12 @@ public class TestTermsEnum extends LuceneTestCase { PostingsEnum de; assertEquals("", te.next().utf8ToString()); - de = te.postings(null, PostingsEnum.NONE); + de = te.postings(null, null, PostingsEnum.NONE); assertEquals(0, de.nextDoc()); assertEquals(1, de.nextDoc()); assertEquals("abc", te.next().utf8ToString()); - de = te.postings(null, PostingsEnum.NONE); + de = te.postings(null, null, PostingsEnum.NONE); assertEquals(0, de.nextDoc()); assertEquals(1, de.nextDoc()); @@ -880,7 +880,7 @@ public class TestTermsEnum extends LuceneTestCase { te = terms.intersect(ca, new BytesRef("")); assertEquals("abc", te.next().utf8ToString()); - de = te.postings(null, PostingsEnum.NONE); + de = te.postings(null, null, PostingsEnum.NONE); assertEquals(0, de.nextDoc()); assertEquals(1, de.nextDoc()); @@ -942,7 +942,7 @@ public class TestTermsEnum extends LuceneTestCase { boolean actualResult = termsEnum.seekExact(termBytesRef); assertEquals(shouldExist, actualResult); if (shouldExist) { - postingsEnum = termsEnum.postings(postingsEnum, 0); + postingsEnum = termsEnum.postings(postingsEnum, null, 0); int docID = postingsEnum.nextDoc(); assertTrue(docID != PostingsEnum.NO_MORE_DOCS); assertEquals(docID, pkLookup.lookup(termBytesRef)); diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java index 00f9b72..83c5edf 100644 --- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java +++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java @@ -387,7 +387,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { TermsEnum termsEnum = terms.iterator(); if (termsEnum.seekExact(catTerm)) { // liveDocs=null because the taxonomy has no deletes - docs = termsEnum.postings(docs, 0 /* freqs not required */); + docs = termsEnum.postings(docs, null, 0 /* freqs not required */); // if the term was found, we know it has exactly one document. doc = docs.nextDoc() + ctx.docBase; break; @@ -705,7 +705,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { // is sufficient to call next(), and then doc(), exactly once with no // 'validation' checks. FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(t.utf8ToString())); - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.NONE); boolean res = cache.put(cp, postingsEnum.nextDoc() + ctx.docBase); assert !res : "entries should not have been evicted from the cache"; } else { @@ -795,7 +795,7 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter { while (te.next() != null) { FacetLabel cp = new FacetLabel(FacetsConfig.stringToPath(te.term().utf8ToString())); final int ordinal = addCategory(cp); - docs = te.postings(docs, PostingsEnum.NONE); + docs = te.postings(docs, null, PostingsEnum.NONE); ordinalMap.addMapping(docs.nextDoc() + base, ordinal); } base += ar.maxDoc(); // no deletions, so we're ok diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java index 135bfe8..1dc8c8e 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java @@ -147,7 +147,7 @@ public final class TokenStreamFromTermVector extends TokenStream { final int termCharsOff = termCharsBuilder.length(); termCharsBuilder.append(tempCharsRefBuilder.chars(), 0, termCharsLen); - dpEnum = termsEnum.postings(dpEnum, dpEnumFlags); + dpEnum = termsEnum.postings(dpEnum, null, dpEnumFlags); assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlier dpEnum.nextDoc(); final int freq = dpEnum.freq(); diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java index 155f0a7..6f67485 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/FieldOffsetStrategy.java @@ -85,7 +85,7 @@ public abstract class FieldOffsetStrategy { TermsEnum termsEnum = termsIndex.iterator();//does not return null for (BytesRef term : sourceTerms) { if (termsEnum.seekExact(term)) { - PostingsEnum postingsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + PostingsEnum postingsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); if (postingsEnum == null) { // no offsets or positions available @@ -124,7 +124,7 @@ public abstract class FieldOffsetStrategy { CharacterRunAutomaton automaton = automata[i]; refBuilder.copyUTF8Bytes(term); if (automaton.run(refBuilder.chars(), 0, refBuilder.length())) { - PostingsEnum postings = termsEnum.postings(null, PostingsEnum.OFFSETS); + PostingsEnum postings = termsEnum.postings(null, null, PostingsEnum.OFFSETS); if (doc == postings.advance(doc)) { automataPostings.get(i).add(postings); } diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java index 4165af4..b6b53b0 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/uhighlight/TermVectorFilteredLeafReader.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; @@ -96,9 +97,9 @@ final class TermVectorFilteredLeafReader extends FilterLeafReader { //TODO delegate docFreq & ttf (moveToCurrentTerm() then call on full? @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { moveToCurrentTerm(); - return baseTermsEnum.postings(reuse, flags); + return baseTermsEnum.postings(reuse, scorer, flags); } void moveToCurrentTerm() throws IOException { diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java index f67ba80..fe2fc71 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java @@ -103,7 +103,7 @@ public class FieldTermStack { if (!termSet.contains(term)) { continue; } - dpEnum = termsEnum.postings(dpEnum, PostingsEnum.POSITIONS); + dpEnum = termsEnum.postings(dpEnum, null, PostingsEnum.POSITIONS); dpEnum.nextDoc(); // For weight look here: http://lucene.apache.org/core/3_6_0/api/core/org/apache/lucene/search/DefaultSimilarity.html diff --git a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java index 98bf5b3..d9a26ab 100644 --- a/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java +++ b/lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java @@ -111,7 +111,7 @@ class TermsIncludingScoreQuery extends Query { PostingsEnum postingsEnum = null; for (int i = 0; i < TermsIncludingScoreQuery.this.terms.size(); i++) { if (segmentTermsEnum.seekExact(TermsIncludingScoreQuery.this.terms.get(ords[i], spare))) { - postingsEnum = segmentTermsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = segmentTermsEnum.postings(postingsEnum, null, PostingsEnum.NONE); if (postingsEnum.advance(doc) == doc) { final float score = TermsIncludingScoreQuery.this.scores[ords[i]]; return Explanation.match(score, "Score based on join value " + segmentTermsEnum.term().utf8ToString()); @@ -168,7 +168,7 @@ class TermsIncludingScoreQuery extends Query { PostingsEnum postingsEnum = null; for (int i = 0; i < terms.size(); i++) { if (termsEnum.seekExact(terms.get(ords[i], spare))) { - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.NONE); float score = TermsIncludingScoreQuery.this.scores[ords[i]]; for (int doc = postingsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = postingsEnum.nextDoc()) { matchingDocs.set(doc); @@ -215,7 +215,7 @@ class TermsIncludingScoreQuery extends Query { PostingsEnum postingsEnum = null; for (int i = 0; i < terms.size(); i++) { if (termsEnum.seekExact(terms.get(ords[i], spare))) { - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.NONE); float score = TermsIncludingScoreQuery.this.scores[ords[i]]; for (int doc = postingsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = postingsEnum.nextDoc()) { // I prefer this: diff --git a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java index 0d8d949..76b1fb9 100644 --- a/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java +++ b/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java @@ -42,6 +42,7 @@ import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.SimpleCollector; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; @@ -1416,7 +1417,7 @@ public class MemoryIndex { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { if (reuse == null || !(reuse instanceof MemoryPostingsEnum)) { reuse = new MemoryPostingsEnum(); } diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java index 54f4aa4..b043953 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java @@ -24,6 +24,7 @@ import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IndexInput; @@ -61,7 +62,7 @@ final class IDVersionPostingsReader extends PostingsReaderBase { } @Override - public PostingsEnum postings(FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { SingleDocsEnum docsEnum; if (PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS)) { diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java index fc643d2..30e1980 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsWriter.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PushPostingsWriterBase; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.store.DataOutput; import org.apache.lucene.store.IndexOutput; @@ -78,7 +79,7 @@ final class IDVersionPostingsWriter extends PushPostingsWriterBase { } @Override - public void startTerm() { + public void startTerm(NumericDocValues norms) { lastDocID = -1; } diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java index 0af64d9..34b1be4 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java @@ -23,6 +23,7 @@ import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; @@ -993,7 +994,7 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); @@ -1002,7 +1003,7 @@ public final class IDVersionSegmentTermsEnum extends TermsEnum { //if (DEBUG) { //System.out.println(" state=" + currentFrame.state); //} - return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, flags); + return fr.parent.postingsReader.postings(fr.fieldInfo, currentFrame.state, reuse, scorer, flags); } @Override diff --git a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java index e9187af..d8abc6a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java +++ b/lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/VersionBlockTreeTermsWriter.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsWriterBase; import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter; import org.apache.lucene.index.FieldInfo; @@ -221,7 +222,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { String lastField = null; for(String field : fields) { @@ -730,7 +731,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer { /** Writes one term's worth of postings. */ public void write(BytesRef text, TermsEnum termsEnum) throws IOException { - BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen); + BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen, null); // TODO: LUCENE-5693: we don't need this check if we fix IW to not send deleted docs to us on flush: if (state != null && ((IDVersionPostingsWriter) postingsWriter).lastDocID != -1) { assert state.docFreq != 0; diff --git a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java index d5607da..4bcc93a 100644 --- a/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java +++ b/lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java @@ -391,7 +391,7 @@ public class TermAutomatonQuery extends Query { if (state != null) { TermsEnum termsEnum = context.reader().terms(field).iterator(); termsEnum.seekExact(term, state); - enums[ent.getKey()] = new EnumAndScorer(ent.getKey(), termsEnum.postings(null, PostingsEnum.POSITIONS)); + enums[ent.getKey()] = new EnumAndScorer(ent.getKey(), termsEnum.postings(null, null, PostingsEnum.POSITIONS)); any = true; } } diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java index 413d401..c1e3fae 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java @@ -283,7 +283,7 @@ public class BlendedInfixSuggester extends AnalyzingInfixSuggester { if (matchedTokens.contains(docTerm) || (prefixToken != null && docTerm.startsWith(prefixToken))) { - PostingsEnum docPosEnum = it.postings(null, PostingsEnum.OFFSETS); + PostingsEnum docPosEnum = it.postings(null, null, PostingsEnum.OFFSETS); docPosEnum.nextDoc(); // use the first occurrence of the term diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java index 9df9d60..05aa01b 100644 --- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java +++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/CompletionFieldsConsumer.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -44,7 +45,7 @@ import static org.apache.lucene.search.suggest.document.CompletionPostingsFormat /** *

- * Weighted FSTs for any indexed {@link SuggestField} is built on {@link #write(Fields)}. + * Weighted FSTs for any indexed {@link SuggestField} is built on {@link #write(Fields,NormsProducer)}. * A weighted FST maps the analyzed forms of a field to its * surface form and document id. FSTs are stored in the CompletionDictionary (.lkp). *

@@ -80,8 +81,8 @@ final class CompletionFieldsConsumer extends FieldsConsumer { } @Override - public void write(Fields fields) throws IOException { - delegateFieldsConsumer.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + delegateFieldsConsumer.write(fields, norms); for (String field : fields) { CompletionTermWriter termWriter = new CompletionTermWriter(); @@ -199,7 +200,7 @@ final class CompletionFieldsConsumer extends FieldsConsumer { * Writes all postings (surface form, weight, document id) for term */ public void write(BytesRef term, TermsEnum termsEnum) throws IOException { - postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.PAYLOADS); + postingsEnum = termsEnum.postings(postingsEnum, null, PostingsEnum.PAYLOADS); builder.startTerm(term); int docFreq = 0; while (postingsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java index a89b508..89e0c33 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java @@ -22,6 +22,7 @@ import java.util.Iterator; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.AssertingLeafReader; import org.apache.lucene.index.FieldInfo; @@ -133,8 +134,8 @@ public final class AssertingPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { - in.write(fields); + public void write(Fields fields, NormsProducer norms) throws IOException { + in.write(fields, norms); // TODO: more asserts? can we somehow run a // "limited" CheckIndex here??? Or ... can we improve @@ -186,7 +187,7 @@ public final class AssertingPostingsFormat extends PostingsFormat { if (hasFreqs) { flags = flags | PostingsEnum.FREQS; } - postingsEnum = termsEnum.postings(postingsEnum, flags); + postingsEnum = termsEnum.postings(postingsEnum, null, flags); } else { flags = PostingsEnum.POSITIONS; if (hasPayloads) { @@ -195,7 +196,7 @@ public final class AssertingPostingsFormat extends PostingsFormat { if (hasOffsets) { flags = flags | PostingsEnum.OFFSETS; } - postingsEnum = termsEnum.postings(postingsEnum, flags); + postingsEnum = termsEnum.postings(postingsEnum, null, flags); } assert postingsEnum != null : "termsEnum=" + termsEnum + " hasPositions=" + hasPositions; diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java index 2ca1bc7..00e168b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/cranky/CrankyPostingsFormat.java @@ -21,6 +21,7 @@ import java.util.Random; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.Fields; import org.apache.lucene.index.SegmentReadState; @@ -61,11 +62,11 @@ class CrankyPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { if (random.nextInt(100) == 0) { throw new IOException("Fake IOException from FieldsConsumer.write()"); } - delegate.write(fields); + delegate.write(fields, norms); } @Override diff --git a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java index 4b85f13..c6d6d2d 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java +++ b/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.PostingsEnum; @@ -42,6 +43,7 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Accountable; @@ -227,7 +229,7 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat { } @Override - public void write(Fields fields) throws IOException { + public void write(Fields fields, NormsProducer norms) throws IOException { for(String field : fields) { Terms terms = fields.terms(field); @@ -282,7 +284,7 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat { break; } RAMPostingsWriterImpl postingsWriter = termsConsumer.startTerm(term); - postingsEnum = termsEnum.postings(postingsEnum, enumFlags); + postingsEnum = termsEnum.postings(postingsEnum, null, enumFlags); int docFreq = 0; long totalTermFreq = 0; @@ -468,7 +470,7 @@ public final class RAMOnlyPostingsFormat extends PostingsFormat { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) { return new RAMDocsEnum(ramField.termToDocs.get(current)); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java index c87697b..0f81707 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/AssertingLeafReader.java @@ -23,6 +23,7 @@ import java.util.Objects; import org.apache.lucene.index.PointValues.IntersectVisitor; import org.apache.lucene.index.PointValues.Relation; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.StringHelper; @@ -187,7 +188,7 @@ public class AssertingLeafReader extends FilterLeafReader { } @Override - public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + public PostingsEnum postings(PostingsEnum reuse, SimScorer scorer, int flags) throws IOException { assertThread("Terms enums", creationThread); assert state == State.POSITIONED: "docs(...) called on unpositioned TermsEnum"; @@ -198,7 +199,7 @@ public class AssertingLeafReader extends FilterLeafReader { } else { actualReuse = null; } - PostingsEnum docs = super.postings(actualReuse, flags); + PostingsEnum docs = super.postings(actualReuse, scorer, flags); assert docs != null; if (docs == actualReuse) { // codec reused, reset asserting state diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java index ab92946..f5b5223 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseIndexFileFormatTestCase.java @@ -334,8 +334,30 @@ abstract class BaseIndexFileFormatTestCase extends LuceneTestCase { SegmentReadState readState = new SegmentReadState(dir, segmentInfo, fieldInfos, IOContext.READ); // PostingsFormat + NormsProducer fakeNorms = new NormsProducer() { + + @Override + public void close() throws IOException {} + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public NumericDocValues getNorms(FieldInfo field) throws IOException { + if (field.hasNorms() == false) { + return null; + } + return oneDocReader.getNormValues(field.name); + } + + @Override + public void checkIntegrity() throws IOException {} + + }; try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState)) { - consumer.write(MultiFields.getFields(oneDocReader)); + consumer.write(MultiFields.getFields(oneDocReader), fakeNorms); IOUtils.close(consumer); IOUtils.close(consumer); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java index f69ca55..2ad2430 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java @@ -32,6 +32,7 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.NormsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -174,8 +175,8 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest } protected static void checkReuse(TermsEnum termsEnum, int firstFlags, int secondFlags, boolean shouldReuse) throws IOException { - PostingsEnum postings1 = termsEnum.postings(null, firstFlags); - PostingsEnum postings2 = termsEnum.postings(postings1, secondFlags); + PostingsEnum postings1 = termsEnum.postings(null, null, firstFlags); + PostingsEnum postings2 = termsEnum.postings(postings1, null, secondFlags); if (shouldReuse) { assertSame("Expected PostingsEnum " + postings1.getClass().getName() + " to be reused", postings1, postings2); } else { @@ -240,7 +241,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest LeafReader ar = getOnlyLeafReader(ir); TermsEnum termsEnum = ar.terms("field").iterator(); assertTrue(termsEnum.seekExact(new BytesRef("value"))); - PostingsEnum docsEnum = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsEnum = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(0, docsEnum.nextDoc()); assertEquals(1, docsEnum.freq()); assertEquals(1, docsEnum.nextDoc()); @@ -263,7 +264,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest LeafReader ar = getOnlyLeafReader(ir); TermsEnum termsEnum = ar.terms("field").iterator(); assertTrue(termsEnum.seekExact(new BytesRef("value"))); - PostingsEnum docsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(0, docsEnum.nextDoc()); assertEquals(1, docsEnum.freq()); assertEquals(1, docsEnum.nextDoc()); @@ -398,8 +399,8 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest return new FieldsConsumer() { @Override - public void write(Fields fields) throws IOException { - fieldsConsumer.write(fields); + public void write(Fields fields, NormsProducer fakeNorms) throws IOException { + fieldsConsumer.write(fields, fakeNorms); boolean isMerge = state.context.context == IOContext.Context.MERGE; @@ -427,9 +428,9 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest // TODO: also sometimes ask for payloads/offsets? boolean noPositions = random().nextBoolean(); if (noPositions) { - docs = termsEnum.postings(docs, PostingsEnum.FREQS); + docs = termsEnum.postings(docs, null, PostingsEnum.FREQS); } else { - docs = termsEnum.postings(null, PostingsEnum.POSITIONS); + docs = termsEnum.postings(null, null, PostingsEnum.POSITIONS); } int docFreq = 0; long totalTermFreq = 0; @@ -476,9 +477,9 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest // TODO: also sometimes ask for payloads/offsets? boolean noPositions = random().nextBoolean(); if (noPositions) { - docs = termsEnum.postings(docs, PostingsEnum.FREQS); + docs = termsEnum.postings(docs, null, PostingsEnum.FREQS); } else { - docs = termsEnum.postings(null, PostingsEnum.POSITIONS); + docs = termsEnum.postings(null, null, PostingsEnum.POSITIONS); } int docFreq = 0; @@ -618,13 +619,13 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest // asking for any flags: ok for (int flag : new int[] { NONE, FREQS, POSITIONS, PAYLOADS, OFFSETS, ALL }) { - postings = termsEnum.postings(null, flag); + postings = termsEnum.postings(null, null, flag); assertEquals(-1, postings.docID()); assertEquals(0, postings.nextDoc()); assertEquals(1, postings.freq()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings.nextDoc()); // reuse that too - postings2 = termsEnum.postings(postings, flag); + postings2 = termsEnum.postings(postings, null, flag); assertNotNull(postings2); assertReused("foo", postings, postings2); // and it had better work @@ -675,14 +676,14 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); assertReused("foo", docsOnly, docsOnly2); // and it had better work @@ -694,7 +695,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest // asking for any flags: ok for (int flag : new int[] { NONE, FREQS, POSITIONS, PAYLOADS, OFFSETS, ALL }) { - postings = termsEnum.postings(null, flag); + postings = termsEnum.postings(null, null, flag); assertEquals(-1, postings.docID()); assertEquals(0, postings.nextDoc()); if (flag != NONE) { @@ -702,7 +703,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest } assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings.nextDoc()); // reuse that too - postings2 = termsEnum.postings(postings, flag); + postings2 = termsEnum.postings(postings, null, flag); assertNotNull(postings2); assertReused("foo", postings, postings2); // and it had better work @@ -753,14 +754,14 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); assertReused("foo", docsOnly, docsOnly2); // and it had better work @@ -786,7 +787,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -818,7 +819,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -848,7 +849,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -877,7 +878,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(-1, docsAndPositionsEnum.endOffset()); assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -933,14 +934,14 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); assertReused("foo", docsOnly, docsOnly2); // and it had better work @@ -968,7 +969,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1004,7 +1005,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1036,7 +1037,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1065,7 +1066,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(7, docsAndPositionsEnum.endOffset()); assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1118,14 +1119,14 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); assertReused("foo", docsOnly, docsOnly2); // and it had better work @@ -1153,7 +1154,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1186,7 +1187,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1218,7 +1219,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertTrue(docsAndPositionsEnum.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1249,7 +1250,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(-1, docsAndPositionsEnum.endOffset()); assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1304,14 +1305,14 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); assertReused("foo", docsOnly, docsOnly2); // and it had better work @@ -1341,7 +1342,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1378,7 +1379,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1412,7 +1413,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertTrue(docsAndPositionsEnum.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); @@ -1443,7 +1444,7 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest assertEquals(7, docsAndPositionsEnum.endOffset()); assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertReused("foo", docsAndPositionsEnum, docsAndPositionsEnum2); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java index 7acee87..26eaee7 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java @@ -455,7 +455,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT this.docsEnum.set(postingsEnum); PostingsEnum docsAndPositionsEnum = termsEnum.postings(null); - docsAndPositionsEnum = termsEnum.postings(random().nextBoolean() ? null : docsAndPositionsEnum, PostingsEnum.POSITIONS); + docsAndPositionsEnum = termsEnum.postings(random().nextBoolean() ? null : docsAndPositionsEnum, null, PostingsEnum.POSITIONS); if (terms.hasPositions() || terms.hasOffsets()) { assertEquals(0, docsAndPositionsEnum.nextDoc()); final int freq = docsAndPositionsEnum.freq(); @@ -779,14 +779,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -797,7 +797,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT // asking for any flags: ok for (int flag : new int[] { NONE, FREQS, POSITIONS, PAYLOADS, OFFSETS, ALL }) { - postings = termsEnum.postings(null, flag); + postings = termsEnum.postings(null, null, flag); assertEquals(-1, postings.docID()); assertEquals(0, postings.nextDoc()); if (flag != NONE) { @@ -805,7 +805,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT } assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings.nextDoc()); // reuse that too - postings2 = termsEnum.postings(postings, flag); + postings2 = termsEnum.postings(postings, null, flag); assertNotNull(postings2); // and it had better work assertEquals(-1, postings2.docID()); @@ -860,14 +860,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -877,7 +877,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly2.nextDoc()); // asking for positions, ok - PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); assertEquals(2, docsAndPositionsEnum.freq()); @@ -892,7 +892,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -907,7 +907,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); // payloads, offsets, etc don't cause an error if they aren't there - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.PAYLOADS); assertNotNull(docsAndPositionsEnum); // but make sure they work assertEquals(-1, docsAndPositionsEnum.docID()); @@ -923,7 +923,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -937,7 +937,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -952,7 +952,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -966,7 +966,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -980,7 +980,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(-1, docsAndPositionsEnum.endOffset()); assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1039,14 +1039,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -1056,7 +1056,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly2.nextDoc()); // asking for positions, ok - PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); assertEquals(2, docsAndPositionsEnum.freq()); @@ -1073,7 +1073,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1090,7 +1090,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); // payloads don't cause an error if they aren't there - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.PAYLOADS); assertNotNull(docsAndPositionsEnum); // but make sure they work assertEquals(-1, docsAndPositionsEnum.docID()); @@ -1108,7 +1108,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1124,7 +1124,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1139,7 +1139,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1153,7 +1153,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1167,7 +1167,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(7, docsAndPositionsEnum.endOffset()); assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1225,14 +1225,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -1242,7 +1242,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly2.nextDoc()); // asking for positions, ok - PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); assertEquals(2, docsAndPositionsEnum.freq()); @@ -1259,7 +1259,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1276,7 +1276,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); // payloads don't cause an error if they aren't there - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.PAYLOADS); assertNotNull(docsAndPositionsEnum); // but make sure they work assertEquals(-1, docsAndPositionsEnum.docID()); @@ -1294,7 +1294,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1310,7 +1310,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1325,7 +1325,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1339,7 +1339,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertNull(docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1353,7 +1353,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(7, docsAndPositionsEnum.endOffset()); assertNull(docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1411,14 +1411,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -1428,7 +1428,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly2.nextDoc()); // asking for positions, ok - PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); assertEquals(2, docsAndPositionsEnum.freq()); @@ -1445,7 +1445,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1462,7 +1462,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); // payloads - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.PAYLOADS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1477,7 +1477,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1491,7 +1491,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(new BytesRef("pay2"), docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1508,7 +1508,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertTrue(docsAndPositionsEnum.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1524,7 +1524,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertTrue(docsAndPositionsEnum2.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum2.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1538,7 +1538,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(-1, docsAndPositionsEnum.endOffset()); assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1597,14 +1597,14 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, postings2.nextDoc()); // asking for docs only: ok - PostingsEnum docsOnly = termsEnum.postings(null, PostingsEnum.NONE); + PostingsEnum docsOnly = termsEnum.postings(null, null, PostingsEnum.NONE); assertEquals(-1, docsOnly.docID()); assertEquals(0, docsOnly.nextDoc()); // we don't define what it is, but if its something else, we should look into it? assertTrue(docsOnly.freq() == 1 || docsOnly.freq() == 2); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly.nextDoc()); // reuse that too - PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, PostingsEnum.NONE); + PostingsEnum docsOnly2 = termsEnum.postings(docsOnly, null, PostingsEnum.NONE); assertNotNull(docsOnly2); // and it had better work assertEquals(-1, docsOnly2.docID()); @@ -1614,7 +1614,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsOnly2.nextDoc()); // asking for positions, ok - PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); assertEquals(2, docsAndPositionsEnum.freq()); @@ -1633,7 +1633,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // now reuse the positions - PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.POSITIONS); + PostingsEnum docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.POSITIONS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1652,7 +1652,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); // payloads - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.PAYLOADS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.PAYLOADS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1669,7 +1669,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.PAYLOADS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.PAYLOADS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1685,7 +1685,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(new BytesRef("pay2"), docsAndPositionsEnum2.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.OFFSETS); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.OFFSETS); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1702,7 +1702,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertTrue(docsAndPositionsEnum.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); // reuse - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.OFFSETS); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.OFFSETS); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); @@ -1718,7 +1718,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertTrue(docsAndPositionsEnum2.getPayload() == null || new BytesRef("pay2").equals(docsAndPositionsEnum2.getPayload())); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum2.nextDoc()); - docsAndPositionsEnum = termsEnum.postings(null, PostingsEnum.ALL); + docsAndPositionsEnum = termsEnum.postings(null, null, PostingsEnum.ALL); assertNotNull(docsAndPositionsEnum); assertEquals(-1, docsAndPositionsEnum.docID()); assertEquals(0, docsAndPositionsEnum.nextDoc()); @@ -1732,7 +1732,7 @@ public abstract class BaseTermVectorsFormatTestCase extends BaseIndexFileFormatT assertEquals(7, docsAndPositionsEnum.endOffset()); assertEquals(new BytesRef("pay2"), docsAndPositionsEnum.getPayload()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); - docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, PostingsEnum.ALL); + docsAndPositionsEnum2 = termsEnum.postings(docsAndPositionsEnum, null, PostingsEnum.ALL); assertEquals(-1, docsAndPositionsEnum2.docID()); assertEquals(0, docsAndPositionsEnum2.nextDoc()); assertEquals(2, docsAndPositionsEnum2.freq()); diff --git a/lucene/test-framework/src/java/org/apache/lucene/index/PerThreadPKLookup.java b/lucene/test-framework/src/java/org/apache/lucene/index/PerThreadPKLookup.java index c417a88..025c732 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/index/PerThreadPKLookup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/index/PerThreadPKLookup.java @@ -77,7 +77,7 @@ public class PerThreadPKLookup { public int lookup(BytesRef id) throws IOException { for(int seg=0;seg= segmentInfo.maxDoc() ? DocIdSetIterator.NO_MORE_DOCS : target; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return true; + } + + @Override + public long longValue() throws IOException { + return doc & 0x0f; + } + }; + } else { + return null; + } + } + + @Override + public void checkIntegrity() throws IOException {} + + }; FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(writeState); boolean success = false; try { - consumer.write(seedFields); + consumer.write(seedFields, fakeNorms); success = true; } finally { if (success) { @@ -749,7 +811,7 @@ public class RandomPostingsTester { System.out.println(" get DocsEnum (but we won't check positions) flags=" + flags); } - threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, flags); + threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, null, flags); postingsEnum = threadState.reusePostingsEnum; } else { if (LuceneTestCase.VERBOSE) { @@ -758,7 +820,7 @@ public class RandomPostingsTester { if (options.contains(Option.REUSE_ENUMS) && random.nextInt(10) < 9) { prevPostingsEnum = threadState.reusePostingsEnum; } - threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, doCheckFreqs ? PostingsEnum.FREQS : PostingsEnum.NONE); + threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, null, doCheckFreqs ? PostingsEnum.FREQS : PostingsEnum.NONE); postingsEnum = threadState.reusePostingsEnum; } } else { @@ -778,7 +840,7 @@ public class RandomPostingsTester { System.out.println(" get DocsEnum flags=" + flags); } - threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, flags); + threadState.reusePostingsEnum = termsEnum.postings(prevPostingsEnum, null, flags); postingsEnum = threadState.reusePostingsEnum; } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java index 3d8d346..ef690e5 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java @@ -2074,12 +2074,12 @@ public abstract class LuceneTestCase extends Assert { assertEquals(info, term, rightTermsEnum.next()); assertTermStatsEquals(info, leftTermsEnum, rightTermsEnum); if (deep) { - assertDocsAndPositionsEnumEquals(info, leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + assertDocsAndPositionsEnumEquals(info, leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.ALL)); assertPositionsSkippingEquals(info, leftReader, leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.postings(leftPositions, PostingsEnum.ALL), - rightPositions = rightTermsEnum.postings(rightPositions, PostingsEnum.ALL)); + leftPositions = leftTermsEnum.postings(leftPositions, null, PostingsEnum.ALL), + rightPositions = rightTermsEnum.postings(rightPositions, null, PostingsEnum.ALL)); // with freqs: @@ -2089,8 +2089,8 @@ public abstract class LuceneTestCase extends Assert { // w/o freqs: - assertDocsEnumEquals(info, leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), + assertDocsEnumEquals(info, leftDocs = leftTermsEnum.postings(leftDocs, null, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, null, PostingsEnum.NONE), false); @@ -2102,8 +2102,8 @@ public abstract class LuceneTestCase extends Assert { // w/o freqs: assertDocsSkippingEquals(info, leftReader, leftTermsEnum.docFreq(), - leftDocs = leftTermsEnum.postings(leftDocs, PostingsEnum.NONE), - rightDocs = rightTermsEnum.postings(rightDocs, PostingsEnum.NONE), + leftDocs = leftTermsEnum.postings(leftDocs, null, PostingsEnum.NONE), + rightDocs = rightTermsEnum.postings(rightDocs, null, PostingsEnum.NONE), false); } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java index 7f53017..2fa2d4e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java +++ b/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java @@ -1127,11 +1127,11 @@ public final class TestUtil { case 2: posFlags = PostingsEnum.PAYLOADS; break; default: posFlags = PostingsEnum.ALL; break; } - return termsEnum.postings(null, posFlags); + return termsEnum.postings(null, null, posFlags); } flags |= PostingsEnum.FREQS; } - return termsEnum.postings(reuse, flags); + return termsEnum.postings(reuse, null, flags); } public static CharSequence stringToCharSequence(String string, Random random) { diff --git a/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java b/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java index 5295349..605d44e 100644 --- a/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java +++ b/lucene/test-framework/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java @@ -320,7 +320,7 @@ public class TestMockAnalyzer extends BaseTokenStreamTestCase { final Terms terms = fields.terms("f"); final TermsEnum te = terms.iterator(); assertEquals(new BytesRef("a"), te.next()); - final PostingsEnum dpe = te.postings(null, PostingsEnum.ALL); + final PostingsEnum dpe = te.postings(null, null, PostingsEnum.ALL); assertEquals(0, dpe.nextDoc()); assertEquals(2, dpe.freq()); assertEquals(0, dpe.nextPosition());