Index: lucene/common-build.xml =================================================================== --- lucene/common-build.xml (.../trunk) (revision 1643331) +++ lucene/common-build.xml (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/common-build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/common-build.xml:r1638800-1642714 Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (.../trunk) (revision 1643331) +++ lucene/CHANGES.txt (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/CHANGES.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/CHANGES.txt:r1638800-1643324 Index: lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java =================================================================== --- lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java (.../trunk) (revision 1643331) +++ lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/BlendedInfixSuggester.java (.../branches/lucene2878) (revision 1643331) @@ -29,7 +29,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.MultiDocValues; import org.apache.lucene.index.Terms; @@ -243,7 +243,7 @@ if (matchedTokens.contains(docTerm) || docTerm.startsWith(prefixToken)) { - DocsAndPositionsEnum docPosEnum = it.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_OFFSETS); + DocsEnum docPosEnum = it.docsAndPositions(null, null, DocsEnum.FLAG_OFFSETS); docPosEnum.nextDoc(); // use the first occurrence of the term Index: lucene/suggest =================================================================== --- lucene/suggest (.../trunk) (revision 1643331) +++ lucene/suggest (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/suggest ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/suggest:r1638800-1642714 Index: lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50DocValuesProducer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50DocValuesProducer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50DocValuesProducer.java (.../branches/lucene2878) (revision 1643331) @@ -17,41 +17,11 @@ * limitations under the License. */ -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_PREFIX_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.CONST_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.DELTA_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.GCD_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.MONOTONIC_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.SORTED_SINGLE_VALUED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.SORTED_WITH_ADDRESSES; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.TABLE_COMPRESSED; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_SHIFT; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_COUNT; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_MASK; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.REVERSE_INTERVAL_SHIFT; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.REVERSE_INTERVAL_MASK; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BLOCK_INTERVAL_SHIFT; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BLOCK_INTERVAL_MASK; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.ALL_LIVE; -import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.ALL_MISSING; - -import java.io.Closeable; // javadocs -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -77,6 +47,35 @@ import org.apache.lucene.util.packed.DirectReader; import org.apache.lucene.util.packed.MonotonicBlockPackedReader; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.ALL_LIVE; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.ALL_MISSING; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_FIXED_UNCOMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_PREFIX_COMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BINARY_VARIABLE_UNCOMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BLOCK_INTERVAL_MASK; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.BLOCK_INTERVAL_SHIFT; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.CONST_COMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.DELTA_COMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.GCD_COMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_COUNT; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_MASK; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.INTERVAL_SHIFT; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.MONOTONIC_COMPRESSED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.REVERSE_INTERVAL_MASK; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.REVERSE_INTERVAL_SHIFT; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.SORTED_SINGLE_VALUED; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.SORTED_WITH_ADDRESSES; +import static org.apache.lucene.codecs.lucene50.Lucene50DocValuesConsumer.TABLE_COMPRESSED; + /** reader for {@link Lucene50DocValuesFormat} */ class Lucene50DocValuesProducer extends DocValuesProducer implements Closeable { private final Map numerics = new HashMap<>(); @@ -1145,7 +1144,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { throw new UnsupportedOperationException(); } } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50PostingsReader.java (.../branches/lucene2878) (revision 1643331) @@ -24,7 +24,6 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat.IntBlockTermState; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; @@ -194,30 +193,38 @@ @Override public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException { - BlockDocsEnum docsEnum; - if (reuse instanceof BlockDocsEnum) { - docsEnum = (BlockDocsEnum) reuse; - if (!docsEnum.canReuse(docIn, fieldInfo)) { + if ((flags & DocsEnum.FLAG_POSITIONS) < DocsEnum.FLAG_POSITIONS) { + BlockDocsEnum docsEnum; + if (reuse instanceof BlockDocsEnum) { + docsEnum = (BlockDocsEnum) reuse; + if (!docsEnum.canReuse(docIn, fieldInfo)) { + docsEnum = new BlockDocsEnum(fieldInfo); + } + } else { docsEnum = new BlockDocsEnum(fieldInfo); } - } else { - docsEnum = new BlockDocsEnum(fieldInfo); + return docsEnum.reset(liveDocs, (IntBlockTermState) termState, flags); } - return docsEnum.reset(liveDocs, (IntBlockTermState) termState, flags); + + return docsAndPositions(fieldInfo, termState, liveDocs, reuse, flags); } // TODO: specialize to liveDocs vs not @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, - DocsAndPositionsEnum reuse, int flags) + public DocsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, + DocsEnum reuse, int flags) throws IOException { + boolean indexHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; boolean indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; boolean indexHasPayloads = fieldInfo.hasPayloads(); - if ((!indexHasOffsets || (flags & DocsAndPositionsEnum.FLAG_OFFSETS) == 0) && - (!indexHasPayloads || (flags & DocsAndPositionsEnum.FLAG_PAYLOADS) == 0)) { + if (!indexHasPositions) + return null; + + if ((!indexHasOffsets || (flags & DocsEnum.FLAG_OFFSETS) == 0) && + (!indexHasPayloads || (flags & DocsEnum.FLAG_PAYLOADS) == 0)) { BlockDocsAndPositionsEnum docsAndPositionsEnum; if (reuse instanceof BlockDocsAndPositionsEnum) { docsAndPositionsEnum = (BlockDocsAndPositionsEnum) reuse; @@ -337,6 +344,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException("Shouldn't call nextPosition() on BlockDocsEnum"); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int docID() { return doc; } @@ -472,7 +509,7 @@ } - final class BlockDocsAndPositionsEnum extends DocsAndPositionsEnum { + final class BlockDocsAndPositionsEnum extends DocsEnum { private final byte[] encoded; @@ -550,7 +587,7 @@ indexHasPayloads == fieldInfo.hasPayloads(); } - public DocsAndPositionsEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException { + public DocsEnum reset(Bits liveDocs, IntBlockTermState termState) throws IOException { this.liveDocs = liveDocs; docFreq = termState.docFreq; @@ -769,6 +806,10 @@ @Override public int nextPosition() throws IOException { + + if (posPendingCount == 0) + return NO_MORE_POSITIONS; + if (posPendingFP != -1) { posIn.seek(posPendingFP); posPendingFP = -1; @@ -792,6 +833,16 @@ } @Override + public int startPosition() { + return position; + } + + @Override + public int endPosition() { + return position; + } + + @Override public int startOffset() { return -1; } @@ -813,7 +864,7 @@ } // Also handles payloads + offsets - final class EverythingEnum extends DocsAndPositionsEnum { + final class EverythingEnum extends DocsEnum { private final byte[] encoded; @@ -960,8 +1011,8 @@ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset; } - this.needsOffsets = (flags & DocsAndPositionsEnum.FLAG_OFFSETS) != 0; - this.needsPayloads = (flags & DocsAndPositionsEnum.FLAG_PAYLOADS) != 0; + this.needsOffsets = (flags & DocsEnum.FLAG_OFFSETS) != 0; + this.needsPayloads = (flags & DocsEnum.FLAG_PAYLOADS) != 0; doc = -1; accum = 0; @@ -1228,6 +1279,9 @@ @Override public int nextPosition() throws IOException { + if (posPendingCount == 0) + return NO_MORE_POSITIONS; + if (posPendingFP != -1) { posIn.seek(posPendingFP); posPendingFP = -1; @@ -1272,6 +1326,16 @@ } @Override + public int startPosition() { + return position; + } + + @Override + public int endPosition() { + return position; + } + + @Override public int startOffset() { return startOffset; } Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsReader.java (.../branches/lucene2878) (revision 1643331) @@ -21,7 +21,7 @@ import java.io.IOException; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; // javadocs -import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs +import org.apache.lucene.index.DocsEnum; // javadocs import org.apache.lucene.index.Fields; import org.apache.lucene.util.Accountable; @@ -40,7 +40,7 @@ /** Returns term vectors for this document, or null if * term vectors were not indexed. If offsets are * available they are in an {@link OffsetAttribute} - * available from the {@link DocsAndPositionsEnum}. */ + * available from the {@link DocsEnum}. */ public abstract Fields get(int doc) throws IOException; /** Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,7 @@ * limitations under the License. */ -import java.io.Closeable; -import java.io.IOException; -import java.util.Iterator; - -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; @@ -34,6 +30,10 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; + /** * Codec API for writing term vectors: *

@@ -226,7 +226,7 @@ String lastFieldName = null; TermsEnum termsEnum = null; - DocsAndPositionsEnum docsAndPositionsEnum = null; + DocsEnum docsAndPositionsEnum = null; int fieldCount = 0; for(String fieldName : vectors) { @@ -283,7 +283,7 @@ final BytesRef payload = docsAndPositionsEnum.getPayload(); - assert !hasPositions || pos >= 0; + assert !hasPositions || pos >= 0 ; addPosition(pos, startOffset, endOffset, payload); } } Index: lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/blocktree/IntersectTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,9 +17,6 @@ * limitations under the License. */ -import java.io.IOException; - -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.TermState; @@ -36,6 +33,8 @@ import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.Outputs; +import java.io.IOException; + // NOTE: cannot seek! final class IntersectTermsEnum extends TermsEnum { final IndexInput in; @@ -209,7 +208,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits skipDocs, DocsEnum reuse, int flags) throws IOException { if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/blocktree/SegmentTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,7 @@ * limitations under the License. */ -import java.io.IOException; -import java.io.PrintStream; - import org.apache.lucene.codecs.BlockTermState; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.TermState; @@ -36,6 +32,9 @@ import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.Util; +import java.io.IOException; +import java.io.PrintStream; + /** Iterates through terms in this field */ final class SegmentTermsEnum extends TermsEnum { @@ -994,7 +993,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits skipDocs, DocsEnum reuse, int flags) throws IOException { if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (.../branches/lucene2878) (revision 1643331) @@ -17,18 +17,6 @@ * limitations under the License. */ -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.BLOCK_SIZE; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_DAT; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_IDX; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.FLAGS_BITS; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.OFFSETS; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PAYLOADS; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.POSITIONS; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT; -import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START; - import java.io.Closeable; import java.io.IOException; import java.util.Collections; @@ -38,7 +26,6 @@ import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.TermVectorsReader; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -63,6 +50,17 @@ import org.apache.lucene.util.packed.BlockPackedReaderIterator; import org.apache.lucene.util.packed.PackedInts; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.BLOCK_SIZE; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_DAT; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.CODEC_SFX_IDX; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.FLAGS_BITS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.OFFSETS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.PAYLOADS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.POSITIONS; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_EXTENSION; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VECTORS_INDEX_EXTENSION; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_CURRENT; +import static org.apache.lucene.codecs.compressing.CompressingTermVectorsWriter.VERSION_START; /** * {@link TermVectorsReader} for {@link CompressingTermVectorsFormat}. @@ -915,17 +913,17 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { if (positions == null && startOffsets == null) { return null; } // TODO: slightly sheisty - return (DocsAndPositionsEnum) docs(liveDocs, reuse, flags); + return docs(liveDocs, reuse, flags); } } - private static class TVDocsEnum extends DocsAndPositionsEnum { + private static class TVDocsEnum extends DocsEnum { private Bits liveDocs; private int doc = -1; @@ -1000,6 +998,16 @@ } @Override + public int startPosition() throws IOException { + return positions[positionIndex + i]; + } + + @Override + public int endPosition() throws IOException { + return positions[positionIndex + i]; + } + + @Override public int startOffset() throws IOException { checkPosition(); if (startOffsets == null) { Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,6 @@ * limitations under the License. */ -import java.io.Closeable; -import java.io.IOException; - -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; @@ -29,10 +25,13 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Bits; +import java.io.Closeable; +import java.io.IOException; + /** The core terms dictionaries (BlockTermsReader, * BlockTreeTermsReader) interact with a single instance * of this class to manage creation of {@link DocsEnum} and - * {@link DocsAndPositionsEnum} instances. It provides an + * {@link DocsEnum} instances. It provides an * IndexInput (termsIn) where this class may read any * previously stored data that it had written in its * corresponding {@link PostingsWriterBase} at indexing @@ -70,9 +69,10 @@ /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsAndPositionsEnum reuse, + public abstract DocsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse, int flags) throws IOException; - + // nocommit this still has the distinction - no need to remove this as long as we get the interface straight? + /** * Checks consistency of this reader. *

@@ -81,7 +81,7 @@ * @lucene.internal */ public abstract void checkIntegrity() throws IOException; - + @Override public abstract void close() throws IOException; } Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,8 @@ * limitations under the License. */ -import java.io.Closeable; -import java.io.IOException; - import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter; -import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs -import org.apache.lucene.index.DocsEnum; // javadocs +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermsEnum; @@ -31,6 +27,9 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; +import java.io.Closeable; +import java.io.IOException; + /** * Class that plugs into term dictionaries, such as {@link * BlockTreeTermsWriter}, and handles writing postings. @@ -54,8 +53,8 @@ public abstract void init(IndexOutput termsOut, SegmentWriteState state) throws IOException; /** Write all postings for one term; use the provided - * {@link TermsEnum} to pull a {@link DocsEnum} or {@link - * DocsAndPositionsEnum}. This method should not + * {@link TermsEnum} to pull a {@link DocsEnum}. + * This method should not * re-position the {@code TermsEnum}! It is already * positioned on the term that should be written. This * method must set the bit in the provided {@link Index: lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java (.../branches/lucene2878) (revision 1643331) @@ -19,7 +19,6 @@ import java.io.IOException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; @@ -43,7 +42,6 @@ // Reused in writeTerm private DocsEnum docsEnum; - private DocsAndPositionsEnum posEnum; private int enumFlags; /** {@link FieldInfo} of current field being written. */ @@ -103,15 +101,15 @@ enumFlags = DocsEnum.FLAG_FREQS; } else if (writeOffsets == false) { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS; + enumFlags = DocsEnum.FLAG_PAYLOADS; } else { - enumFlags = 0; + enumFlags = DocsEnum.FLAG_POSITIONS; } } else { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS | DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_PAYLOADS | DocsEnum.FLAG_OFFSETS; } else { - enumFlags = DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_OFFSETS; } } @@ -124,8 +122,7 @@ if (writePositions == false) { docsEnum = termsEnum.docs(null, docsEnum, enumFlags); } else { - posEnum = termsEnum.docsAndPositions(null, posEnum, enumFlags); - docsEnum = posEnum; + docsEnum = termsEnum.docsAndPositions(null, docsEnum, enumFlags); } assert docsEnum != null; @@ -149,13 +146,13 @@ if (writePositions) { for(int i=0;i A Token can optionally have metadata (a.k.a. payload) in the form of a variable - length byte array. Use {@link DocsAndPositionsEnum#getPayload()} to retrieve the + length byte array. Use {@link DocsEnum#getPayload()} to retrieve the payloads from the index.

Index: lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java (.../branches/lucene2878) (revision 1643331) @@ -43,7 +43,7 @@ * * * - * @see org.apache.lucene.index.DocsAndPositionsEnum + * @see org.apache.lucene.index.DocsEnum */ public interface PositionIncrementAttribute extends Attribute { /** Set the position increment. The default value is one. Index: lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java (.../branches/lucene2878) (revision 1643331) @@ -17,7 +17,7 @@ * limitations under the License. */ -import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs +import org.apache.lucene.index.DocsEnum; // javadocs import org.apache.lucene.util.Attribute; import org.apache.lucene.util.BytesRef; @@ -33,7 +33,7 @@ * best to use the minimum number of bytes necessary. Some codec implementations * may optimize payload storage when all payloads have the same length. * - * @see DocsAndPositionsEnum + * @see DocsEnum */ public interface PayloadAttribute extends Attribute { /** Index: lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (.../branches/lucene2878) (revision 1643331) @@ -23,18 +23,22 @@ import java.util.Comparator; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; /** Scorer for conjunctions, sets of queries, all of which are required. */ class ConjunctionScorer extends Scorer { + protected int lastDoc = -1; protected final DocsAndFreqs[] docsAndFreqs; private final DocsAndFreqs lead; private final float coord; + private final PositionQueue posQueue; + private final Scorer[] scorers; // to preserve order for positional queries ConjunctionScorer(Weight weight, Scorer[] scorers) { this(weight, scorers, 1f); } - + ConjunctionScorer(Weight weight, Scorer[] scorers, float coord) { super(weight); this.coord = coord; @@ -52,6 +56,8 @@ }); lead = docsAndFreqs[0]; // least frequent DocsEnum leads the intersection + posQueue = new PositionQueue(scorers); + this.scorers = scorers; } private int doNext(int doc) throws IOException { @@ -76,6 +82,7 @@ } } // success - all DocsEnums are on the same doc + posQueue.advanceTo(doc); return doc; } // advance head for next iteration @@ -109,7 +116,7 @@ } return sum * coord; } - + @Override public int freq() { return docsAndFreqs.length; @@ -116,6 +123,36 @@ } @Override + public int nextPosition() throws IOException { + return posQueue.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + return posQueue.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return posQueue.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return posQueue.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return posQueue.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return posQueue.getPayload(); + } + + @Override public long cost() { return lead.scorer.cost(); } @@ -122,9 +159,9 @@ @Override public Collection getChildren() { - ArrayList children = new ArrayList<>(docsAndFreqs.length); - for (DocsAndFreqs docs : docsAndFreqs) { - children.add(new ChildScorer(docs.scorer, "MUST")); + ArrayList children = new ArrayList<>(scorers.length); + for (Scorer scorer : scorers) { + children.add(new ChildScorer(scorer, "MUST")); } return children; } @@ -133,7 +170,7 @@ final long cost; final Scorer scorer; int doc = -1; - + DocsAndFreqs(Scorer scorer) { this.scorer = scorer; this.cost = scorer.cost(); Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (.../branches/lucene2878) (revision 1643331) @@ -20,23 +20,28 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Locale; +import org.apache.lucene.util.BytesRef; + /** * Base class for Scorers that score disjunctions. */ abstract class DisjunctionScorer extends Scorer { - private final Scorer subScorers[]; - private int numScorers; + protected final Scorer subScorers[]; /** The document number of the current match. */ protected int doc = -1; + protected int numScorers; + protected PositionQueue posQueue; /** Number of matching scorers for the current match. */ protected int freq = -1; - + protected DisjunctionScorer(Weight weight, Scorer subScorers[]) { super(weight); this.subScorers = subScorers; this.numScorers = subScorers.length; + this.posQueue = new PositionQueue(subScorers); if (numScorers <= 1) { throw new IllegalArgumentException("There must be at least 2 subScorers"); } @@ -115,6 +120,50 @@ } @Override + public int nextPosition() throws IOException { + //System.out.println("Advancing " + this.toString()); + int pos = posQueue.nextPosition(); + //System.out.println(this); + return pos; + } + + @Override + public int startPosition() throws IOException { + return posQueue.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return posQueue.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return posQueue.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return posQueue.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return posQueue.getPayload(); + } + + @Override + public String toString() { + try { + return String.format(Locale.ROOT, "DisjScorer[%s] %d(%d)->%d(%d)", weight.toString(), + posQueue.startPosition(), + posQueue.startOffset(), posQueue.endPosition(), posQueue.endOffset()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override public final long cost() { long sum = 0; for (int i = 0; i < numScorers; i++) { @@ -143,6 +192,7 @@ int docID = subScorers[0].docID(); if (docID != doc) { freq = -1; + posQueue.advanceTo(docID); return doc = docID; } } @@ -163,11 +213,12 @@ int docID = subScorers[0].docID(); if (docID >= target) { freq = -1; + posQueue.advanceTo(docID); return doc = docID; } } } - + // if we haven't already computed freq + score, do so private void visitScorers() throws IOException { reset(); @@ -209,4 +260,5 @@ /** Return final score */ protected abstract float getFinal(); + } Index: lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/TotalHitCountCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; /** * Just counts the total number of hits. @@ -36,6 +37,15 @@ } @Override + public int postingFeatures() { + // nocommit: we don't need frequencies here, but it causes test failures if + // we don't ask for them because our leafcollectors sometimes get wrapped in + // RandomOrderCollectors, which call score(), and that trips an assertion + // in AssertingLeafReader.freq() when the freq turns out to be 0. + return DocsEnum.FLAG_FREQS; + } + + @Override public boolean acceptsDocsOutOfOrder() { return true; } Index: lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,17 +17,19 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ToStringUtils; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - /** * A query that wraps another query or a filter and simply returns a constant score equal to the * query boost for every document that matches the filter or query. @@ -134,14 +136,14 @@ } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { final DocIdSetIterator disi; if (filter != null) { assert query == null; - return super.bulkScorer(context, scoreDocsInOrder, acceptDocs); + return super.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); } else { assert query != null && innerWeight != null; - BulkScorer bulkScorer = innerWeight.bulkScorer(context, scoreDocsInOrder, acceptDocs); + BulkScorer bulkScorer = innerWeight.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); if (bulkScorer == null) { return null; } @@ -150,8 +152,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { - final DocIdSetIterator disi; + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { if (filter != null) { assert query == null; final DocIdSet dis = filter.getDocIdSet(context, acceptDocs); @@ -158,16 +159,20 @@ if (dis == null) { return null; } - disi = dis.iterator(); + final DocIdSetIterator disi = dis.iterator(); + if (disi == null) + return null; + return new ConstantDocIdSetIteratorScorer(disi, this, queryWeight); } else { assert query != null && innerWeight != null; - disi = innerWeight.scorer(context, acceptDocs); + Scorer scorer = innerWeight.scorer(context, flags, acceptDocs); + if (scorer == null) { + return null; + } + return new ConstantScoreScorer(scorer, queryWeight); } - if (disi == null) { - return null; - } - return new ConstantScorer(disi, this, queryWeight); + } @Override @@ -177,7 +182,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - final Scorer cs = scorer(context, context.reader().getLiveDocs()); + final Scorer cs = scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); final boolean exists = (cs != null && cs.advance(doc) == doc); final ComplexExplanation result = new ComplexExplanation(); @@ -221,17 +226,46 @@ @Override public void setScorer(Scorer scorer) throws IOException { // we must wrap again here, but using the scorer passed in as parameter: - in.setScorer(new ConstantScorer(scorer, weight, theScore)); + in.setScorer(new ConstantScoreScorer(scorer, theScore)); } }; } } - protected class ConstantScorer extends Scorer { + protected class ConstantScoreScorer extends FilterScorer { + + private final float score; + + public ConstantScoreScorer(Scorer wrapped, float score) { + super(wrapped); + this.score = score; + } + + @Override + public int freq() throws IOException { + return 1; + } + + @Override + public float score() throws IOException { + return score; + } + + @Override + public Collection getChildren() { + if (query != null) { + return Collections.singletonList(new ChildScorer(in, "constant")); + } else { + return Collections.emptyList(); + } + } + } + + protected class ConstantDocIdSetIteratorScorer extends Scorer { final DocIdSetIterator docIdSetIterator; final float theScore; - public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) { + public ConstantDocIdSetIteratorScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) { super(w); this.theScore = theScore; this.docIdSetIterator = docIdSetIterator; @@ -259,10 +293,40 @@ } @Override + public int nextPosition() throws IOException { + return NO_MORE_POSITIONS; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public int advance(int target) throws IOException { return docIdSetIterator.advance(target); } - + @Override public long cost() { return docIdSetIterator.cost(); Index: lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/similarities/Similarity.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,8 @@ * limitations under the License. */ -import java.io.IOException; - +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.Explanation; @@ -28,10 +26,12 @@ import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermStatistics; -import org.apache.lucene.search.spans.SpanQuery; // javadoc +import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SmallFloat; // javadoc +import org.apache.lucene.util.SmallFloat; +import java.io.IOException; + /** * Similarity defines the components of Lucene scoring. *

Index: lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,18 +17,18 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; - /** * A query that applies a filter to the results of another query. * @@ -124,7 +124,7 @@ // return a filtering scorer @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { assert filter != null; DocIdSet filterDocIdSet = filter.getDocIdSet(context, acceptDocs); @@ -133,12 +133,12 @@ return null; } - return strategy.filteredScorer(context, weight, filterDocIdSet); + return strategy.filteredScorer(context, weight, filterDocIdSet, flags); } // return a filtering top scorer @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { assert filter != null; DocIdSet filterDocIdSet = filter.getDocIdSet(context, acceptDocs); @@ -147,7 +147,8 @@ return null; } - return strategy.filteredBulkScorer(context, weight, scoreDocsInOrder, filterDocIdSet); + return strategy.filteredBulkScorer(context, weight, scoreDocsInOrder, filterDocIdSet, flags); + } }; } @@ -158,13 +159,13 @@ * than document scoring or if the filter has a linear running time to compute * the next matching doc like exact geo distances. */ - private static final class QueryFirstScorer extends Scorer { + private static final class QueryFirstScorer extends FilterScorer { private final Scorer scorer; private int scorerDoc = -1; private final Bits filterBits; protected QueryFirstScorer(Weight weight, Bits filterBits, Scorer other) { - super(weight); + super(other, weight); this.scorer = other; this.filterBits = filterBits; } @@ -189,29 +190,16 @@ return scorerDoc = doc; } } - @Override public int docID() { return scorerDoc; } - + @Override - public float score() throws IOException { - return scorer.score(); - } - - @Override - public int freq() throws IOException { return scorer.freq(); } - - @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "FILTERED")); } - @Override - public long cost() { - return scorer.cost(); - } } private static class QueryFirstBulkScorer extends BulkScorer { @@ -254,7 +242,7 @@ * jumping past the target document. When both land on the same document, it's * collected. */ - private static final class LeapFrogScorer extends Scorer { + private static final class LeapFrogScorer extends FilterScorer { private final DocIdSetIterator secondary; private final DocIdSetIterator primary; private final Scorer scorer; @@ -262,7 +250,7 @@ private int secondaryDoc = -1; protected LeapFrogScorer(Weight weight, DocIdSetIterator primary, DocIdSetIterator secondary, Scorer scorer) { - super(weight); + super(scorer, weight); this.primary = primary; this.secondary = secondary; this.scorer = scorer; @@ -302,18 +290,8 @@ public final int docID() { return secondaryDoc; } - + @Override - public final float score() throws IOException { - return scorer.score(); - } - - @Override - public final int freq() throws IOException { - return scorer.freq(); - } - - @Override public final Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "FILTERED")); } @@ -460,12 +438,13 @@ * the {@link org.apache.lucene.index.LeafReaderContext} for which to return the {@link Scorer}. * @param weight the {@link FilteredQuery} {@link Weight} to create the filtered scorer. * @param docIdSet the filter {@link DocIdSet} to apply + * @param flags the low level Posting Features for this scorer. * @return a filtered scorer * * @throws IOException if an {@link IOException} occurs */ public abstract Scorer filteredScorer(LeafReaderContext context, - Weight weight, DocIdSet docIdSet) throws IOException; + Weight weight, DocIdSet docIdSet, int flags) throws IOException; /** * Returns a filtered {@link BulkScorer} based on this @@ -480,8 +459,8 @@ * @return a filtered top scorer */ public BulkScorer filteredBulkScorer(LeafReaderContext context, - Weight weight, boolean scoreDocsInOrder, DocIdSet docIdSet) throws IOException { - Scorer scorer = filteredScorer(context, weight, docIdSet); + Weight weight, boolean scoreDocsInOrder, DocIdSet docIdSet, int flags) throws IOException { + Scorer scorer = filteredScorer(context, weight, docIdSet, flags); if (scorer == null) { return null; } @@ -489,6 +468,7 @@ // ignore scoreDocsInOrder: return new Weight.DefaultBulkScorer(scorer); } + } /** @@ -502,7 +482,7 @@ public static class RandomAccessFilterStrategy extends FilterStrategy { @Override - public Scorer filteredScorer(LeafReaderContext context, Weight weight, DocIdSet docIdSet) throws IOException { + public Scorer filteredScorer(LeafReaderContext context, Weight weight, DocIdSet docIdSet, int flags) throws IOException { final DocIdSetIterator filterIter = docIdSet.iterator(); if (filterIter == null) { // this means the filter does not accept any documents. @@ -514,11 +494,11 @@ final boolean useRandomAccess = filterAcceptDocs != null && useRandomAccess(filterAcceptDocs, filterIter.cost()); if (useRandomAccess) { // if we are using random access, we return the inner scorer, just with other acceptDocs - return weight.scorer(context, filterAcceptDocs); + return weight.scorer(context, flags, filterAcceptDocs); } else { // we are gonna advance() this scorer, so we set inorder=true/toplevel=false // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice - final Scorer scorer = weight.scorer(context, null); + final Scorer scorer = weight.scorer(context, flags, null); return (scorer == null) ? null : new LeapFrogScorer(weight, filterIter, scorer, scorer); } } @@ -551,7 +531,7 @@ @Override public Scorer filteredScorer(LeafReaderContext context, - Weight weight, DocIdSet docIdSet) throws IOException { + Weight weight, DocIdSet docIdSet, int flags) throws IOException { final DocIdSetIterator filterIter = docIdSet.iterator(); if (filterIter == null) { // this means the filter does not accept any documents. @@ -558,7 +538,7 @@ return null; } // we pass null as acceptDocs, as our filter has already respected acceptDocs, no need to do twice - final Scorer scorer = weight.scorer(context, null); + final Scorer scorer = weight.scorer(context, flags, null); if (scorer == null) { return null; } @@ -588,16 +568,15 @@ @Override public Scorer filteredScorer(final LeafReaderContext context, Weight weight, - DocIdSet docIdSet) throws IOException { + DocIdSet docIdSet, int flags) throws IOException { Bits filterAcceptDocs = docIdSet.bits(); if (filterAcceptDocs == null) { // Filter does not provide random-access Bits; we // must fallback to leapfrog: - return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredScorer(context, weight, docIdSet); + return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredScorer(context, weight, docIdSet, flags); } - final Scorer scorer = weight.scorer(context, null); - return scorer == null ? null : new QueryFirstScorer(weight, - filterAcceptDocs, scorer); + final Scorer scorer = weight.scorer(context, flags, null); + return scorer == null ? null : new QueryFirstScorer(weight, filterAcceptDocs, scorer); } @Override @@ -604,14 +583,14 @@ public BulkScorer filteredBulkScorer(final LeafReaderContext context, Weight weight, boolean scoreDocsInOrder, // ignored (we always top-score in order) - DocIdSet docIdSet) throws IOException { + DocIdSet docIdSet, int flags) throws IOException { Bits filterAcceptDocs = docIdSet.bits(); if (filterAcceptDocs == null) { // Filter does not provide random-access Bits; we // must fallback to leapfrog: - return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredBulkScorer(context, weight, scoreDocsInOrder, docIdSet); + return LEAP_FROG_QUERY_FIRST_STRATEGY.filteredBulkScorer(context, weight, scoreDocsInOrder, docIdSet, flags); } - final Scorer scorer = weight.scorer(context, null); + final Scorer scorer = weight.scorer(context, flags, null); return scorer == null ? null : new QueryFirstBulkScorer(scorer, filterAcceptDocs); } } Index: lucene/core/src/java/org/apache/lucene/search/Collector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/Collector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/Collector.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,10 @@ * limitations under the License. */ +import org.apache.lucene.index.LeafReaderContext; + import java.io.IOException; -import org.apache.lucene.index.LeafReaderContext; - /** *

Expert: Collectors are primarily meant to be used to * gather raw results from a search, and implement sorting @@ -72,5 +72,10 @@ * next atomic reader context */ LeafCollector getLeafCollector(LeafReaderContext context) throws IOException; - + + /** + * Returns the posting features required by this collector. + */ + public int postingFeatures(); + } Index: lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/PhraseQueue.java (.../branches/lucene2878) (revision 1643331) @@ -29,11 +29,11 @@ if (pp1.doc == pp2.doc) if (pp1.position == pp2.position) // same doc and pp.position, so decide by actual term positions. - // rely on: pp.position == tp.position - offset. - if (pp1.offset == pp2.offset) { + // rely on: pp.position == tp.position - phraseOffset. + if (pp1.phraseOffset == pp2.phraseOffset) { return pp1.ord < pp2.ord; } else { - return pp1.offset < pp2.offset; + return pp1.phraseOffset < pp2.phraseOffset; } else { return pp1.position < pp2.position; @@ -42,4 +42,8 @@ return pp1.doc < pp2.doc; } } + + public Object[] getPPs() { + return getHeapArray(); + } } Index: lucene/core/src/java/org/apache/lucene/search/FilterCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilterCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/FilterCollector.java (.../branches/lucene2878) (revision 1643331) @@ -1,9 +1,9 @@ package org.apache.lucene.search; +import org.apache.lucene.index.LeafReaderContext; + import java.io.IOException; -import org.apache.lucene.index.LeafReaderContext; - /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -41,6 +41,11 @@ } @Override + public int postingFeatures() { + return in.postingFeatures(); + } + + @Override public String toString() { return getClass().getSimpleName() + "(" + in + ")"; } Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (.../branches/lucene2878) (revision 1643331) @@ -19,6 +19,7 @@ import java.io.IOException; + /** A Scorer for OR like queries, counterpart of ConjunctionScorer. * This Scorer implements {@link Scorer#advance(int)} and uses advance() on the given Scorers. */ @@ -25,7 +26,7 @@ final class DisjunctionSumScorer extends DisjunctionScorer { private double score; private final float[] coord; - + /** Construct a DisjunctionScorer. * @param weight The weight to be used. * @param subScorers Array of at least two subscorers. @@ -50,4 +51,5 @@ protected float getFinal() { return (float)score * coord[freq]; } + } Index: lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/TimeLimitingCollector.java (.../branches/lucene2878) (revision 1643331) @@ -156,7 +156,12 @@ }; } - + + @Override + public int postingFeatures() { + return collector.postingFeatures(); + } + /** * This is so the same timer can be used with a multi-phase search process such as grouping. * We don't want to create a new TimeLimitingCollector for each phase because that would Index: lucene/core/src/java/org/apache/lucene/search/FilterScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilterScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/FilterScorer.java (.../branches/lucene2878) (revision 1643331) @@ -18,9 +18,9 @@ */ import java.io.IOException; -import java.util.Collection; import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; /** * A {@code FilterScorer} contains another {@code Scorer}, which it @@ -32,7 +32,7 @@ * further override some of these methods and may also provide additional * methods and fields. */ -abstract class FilterScorer extends Scorer { +public abstract class FilterScorer extends Scorer { protected final Scorer in; public FilterScorer(Scorer in) { @@ -39,6 +39,11 @@ super(in.weight); this.in = in; } + + public FilterScorer(Scorer in, Weight weight) { + super(weight); + this.in = in; + } @Override public float score() throws IOException { @@ -61,6 +66,11 @@ } @Override + public int nextPosition() throws IOException { + return in.nextPosition(); + } + + @Override public int advance(int target) throws IOException { return in.advance(target); } @@ -71,6 +81,31 @@ } @Override + public int startPosition() throws IOException { + return in.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return in.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return in.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return in.getPayload(); + } + + @Override public AttributeSource attributes() { return in.attributes(); } Index: lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (.../branches/lucene2878) (revision 1643331) @@ -29,26 +29,32 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.FixedBitSet; -final class SloppyPhraseScorer extends Scorer { + +final class SloppyPhraseScorer extends PhraseScorer { private PhrasePositions min, max; - private float sloppyFreq; //phrase frequency in current doc as computed by phraseFreq(). + private float sloppyFreq; private final Similarity.SimScorer docScorer; + private final PhraseQuery.PostingsAndFreq[] postings; private final int slop; private final int numPostings; private final PhraseQueue pq; // for advancing min position - private int end; // current largest phrase position + private int currentEnd; // current largest phrase position + private int currentRealEnd; // current largest phrase position, including phrase offset + private int currentEndOffset; // current largest phrase currentEnd offset + private int spanEnd; + private int spanOffsetEnd; + private boolean hasRpts; // flag indicating that there are repetitions (as checked in first candidate doc) private boolean checkedRpts; // flag to only check for repetitions in first candidate doc private boolean hasMultiTermRpts; // - private PhrasePositions[][] rptGroups; // in each group are PPs that repeats each other (i.e. same term), sorted by (query) offset + private PhrasePositions[][] rptGroups; // in each group are PPs that repeats each other (i.e. same term), sorted by (query) phraseOffset private PhrasePositions[] rptStack; // temporary stack for switching colliding repeating pps - private int numMatches; private final long cost; SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, @@ -55,17 +61,18 @@ int slop, Similarity.SimScorer docScorer) { super(weight); this.docScorer = docScorer; + this.postings = postings; this.slop = slop; this.numPostings = postings==null ? 0 : postings.length; - pq = new PhraseQueue(postings.length); - // min(cost) - cost = postings[0].postings.cost(); + pq = new PhraseQueue(this.numPostings); // convert tps to a list of phrase positions. // note: phrase-position differs from term-position in that its position - // reflects the phrase offset: pp.pos = tp.pos - offset. + // reflects the phrase phraseOffset: pp.pos = tp.pos - phraseOffset. // this allows to easily identify a matching (exact) phrase // when all PhrasePositions have exactly the same position. - if (postings.length > 0) { + if (postings != null && postings.length > 0) { + // min(cost) + cost = postings[0].postings.cost(); min = new PhrasePositions(postings[0].postings, postings[0].position, 0, postings[0].terms); max = min; max.doc = -1; @@ -77,70 +84,111 @@ } max.next = min; // make it cyclic for easier manipulation } + else { + cost = 0; + } } - /** - * Score a candidate doc for all slop-valid position-combinations (matches) - * encountered while traversing/hopping the PhrasePositions. - *
The score contribution of a match depends on the distance: - *
- highest score for distance=0 (exact match). - *
- score gets lower as distance gets higher. - *
Example: for query "a b"~2, a document "x a b a y" can be scored twice: - * once for "a b" (distance=0), and once for "b a" (distance=2). - *
Possibly not all valid combinations are encountered, because for efficiency - * we always propagate the least PhrasePosition. This allows to base on - * PriorityQueue and move forward faster. - * As result, for example, document "a b c b a" - * would score differently for queries "a b c"~4 and "c b a"~4, although - * they really are equivalent. - * Similarly, for doc "a b c b a f g", query "c b"~2 - * would get same score as "g f"~2, although "c b"~2 could be matched twice. - * We may want to fix this in the future (currently not, for performance reasons). - */ - private float phraseFreq() throws IOException { - if (!initPhrasePositions()) { - return 0.0f; + private int matchLength; + private int startpos = -1; + private int startoffset = -1; + private int endoffset = -1; + + @Override + protected int doNextPosition() throws IOException { + if (cached) { + cached = false; + return this.startPosition(); } - float freq = 0.0f; - numMatches = 0; - PhrasePositions pp = pq.pop(); - int matchLength = end - pp.position; - int next = pq.top().position; - while (advancePP(pp)) { - if (hasRpts && !advanceRpts(pp)) { + + if (pq.size() < postings.length) + return NO_MORE_POSITIONS; + + PhrasePositions top = pq.pop(); + matchLength = currentEnd - top.position; + int next = pq.top().position; + int pos = top.position + top.phraseOffset; + int startoffset = top.startOffset(); + spanEnd = currentRealEnd; + spanOffsetEnd = currentEndOffset; + while (advancePP(top)) { + if (hasRpts && !advanceRpts(top)) break; // pps exhausted - } - if (pp.position > next) { // done minimizing current match-length + if (top.position > next) { // done minimizing current match-length if (matchLength <= slop) { - freq += docScorer.computeSlopFactor(matchLength); // score match - numMatches++; - } - pq.add(pp); - pp = pq.pop(); + setSpanStart(pos, startoffset); + pq.add(top); + return startpos; + } + pq.add(top); + top = pq.pop(); next = pq.top().position; - matchLength = end - pp.position; - } else { - int matchLength2 = end - pp.position; - if (matchLength2 < matchLength) { + matchLength = currentEnd - top.position; + pos = top.position + top.phraseOffset; + startoffset = top.startOffset(); + spanEnd = currentRealEnd; + spanOffsetEnd = currentEndOffset; + } + else { + int matchLength2 = currentEnd - top.position; + pos = top.position + top.phraseOffset; + startoffset = top.startOffset(); + spanEnd = currentRealEnd; + spanOffsetEnd = currentEndOffset; + if (matchLength2 < matchLength) matchLength = matchLength2; - } } } + if (matchLength <= slop) { - freq += docScorer.computeSlopFactor(matchLength); // score match - numMatches++; - } - return freq; + setSpanStart(pos, startoffset); + return startpos; + } + + return NO_MORE_POSITIONS; + } - /** advance a PhrasePosition and update 'end', return false if exhausted */ + private void setSpanStart(int topPos, int topStartOffset) throws IOException { + startpos = topPos; + startoffset = topStartOffset; + for (Object o : pq.getPPs()) { + if (o == null) + continue; + PhrasePositions pp = (PhrasePositions) o; + if (pp.position != NO_MORE_POSITIONS) { + startpos = Math.min(startpos, pp.position + pp.phraseOffset); + startoffset = Math.min(startoffset, pp.startOffset()); + } + } + } + + boolean cached = false; + + private int firstPosition() throws IOException { + if (!initPhrasePositions()) + return NO_MORE_POSITIONS; + + freq = -1; + sloppyFreq = -1; + cached = false; + int pos = doNextPosition(); + cached = true; + return pos; + } + + /** advance a PhrasePosition and update 'currentEnd', return false if exhausted */ private boolean advancePP(PhrasePositions pp) throws IOException { if (!pp.nextPosition()) { return false; } - if (pp.position > end) { - end = pp.position; + if (pp.position > currentEnd) { + currentEnd = pp.position; } + if (pp.position + pp.phraseOffset > currentRealEnd) { + currentRealEnd = pp.position + pp.phraseOffset; + currentEndOffset = pp.endOffset(); + } return true; } @@ -186,10 +234,10 @@ return true; } - /** compare two pps, but only by position and offset */ + /** compare two pps, but only by position and phraseOffset */ private PhrasePositions lesser(PhrasePositions pp, PhrasePositions pp2) { if (pp.position < pp2.position || - (pp.position == pp2.position && pp.offset < pp2.offset)) { + (pp.position == pp2.position && pp.phraseOffset < pp2.phraseOffset)) { return pp; } return pp2; @@ -224,7 +272,8 @@ * @return false if PPs are exhausted (and so current doc will not be a match) */ private boolean initPhrasePositions() throws IOException { - end = Integer.MIN_VALUE; + currentEnd = currentRealEnd = Integer.MIN_VALUE; + currentEndOffset = -1; if (!checkedRpts) { return initFirstTime(); } @@ -242,9 +291,13 @@ // position pps and build queue from list for (PhrasePositions pp=min,prev=null; prev!=max; pp=(prev=pp).next) { // iterate cyclic list: done once handled max pp.firstPosition(); - if (pp.position > end) { - end = pp.position; + if (pp.position > currentEnd) { + currentEnd = pp.position; } + if (pp.position + pp.phraseOffset > currentRealEnd) { + currentRealEnd = pp.position + pp.phraseOffset; + currentEndOffset = pp.endOffset(); + } pq.add(pp); } } @@ -268,17 +321,21 @@ } /** Fill the queue (all pps are already placed */ - private void fillQueue() { + private void fillQueue() throws IOException { pq.clear(); for (PhrasePositions pp=min,prev=null; prev!=max; pp=(prev=pp).next) { // iterate cyclic list: done once handled max - if (pp.position > end) { - end = pp.position; + if (pp.position > currentEnd) { + currentEnd = pp.position; } + if (pp.position + pp.phraseOffset > currentRealEnd) { + currentRealEnd = pp.position + pp.phraseOffset; + currentEndOffset = pp.endOffset(); + } pq.add(pp); } } - /** At initialization (each doc), each repetition group is sorted by (query) offset. + /** At initialization (each doc), each repetition group is sorted by (query) phraseOffset. * This provides the start condition: no collisions. *

Case 1: no multi-term repeats
* It is sufficient to advance each pp in the group by one less than its group index. @@ -298,7 +355,7 @@ int k; while((k=collide(pp)) >= 0) { PhrasePositions pp2 = lesser(pp, rg[k]); - if (!advancePP(pp2)) { // at initialization always advance pp with higher offset + if (!advancePP(pp2)) { // at initialization always advance pp with higher phraseOffset return false; // exhausted } if (pp2.rptInd < i) { // should not happen? @@ -356,7 +413,7 @@ return true; // PPs available } - /** sort each repetition group by (query) offset. + /** sort each repetition group by (query) phraseOffset. * Done only once (at first doc) and allows to initialize faster for each doc. */ private void sortRptGroups(ArrayList> rgs) { rptGroups = new PhrasePositions[rgs.size()][]; @@ -363,7 +420,7 @@ Comparator cmprtr = new Comparator() { @Override public int compare(PhrasePositions pp1, PhrasePositions pp2) { - return pp1.offset - pp2.offset; + return pp1.phraseOffset - pp2.phraseOffset; } }; for (int i=0; i=0 // already marked as a repetition - || pp2.offset == pp.offset // not a repetition: two PPs are originally in same offset in the query! + || pp2.phraseOffset == pp.phraseOffset // not a repetition: two PPs are originally in same phraseOffset in the query! || tpPos(pp2) != tpPos) { // not a repetition continue; } @@ -434,9 +491,9 @@ return res; } - /** Actual position in doc of a PhrasePosition, relies on that position = tpPos - offset) */ - private final int tpPos(PhrasePositions pp) { - return pp.position + pp.offset; + /** Actual position in doc of a PhrasePosition, relies on that position = tpPos - phraseOffset) */ + private int tpPos(PhrasePositions pp) { + return pp.position + pp.phraseOffset; } /** find repeating terms and assign them ordinal values */ @@ -446,7 +503,7 @@ for (PhrasePositions pp=min,prev=null; prev!=max; pp=(prev=pp).next) { // iterate cyclic list: done once handled max for (Term t : pp.terms) { Integer cnt0 = tcnt.get(t); - Integer cnt = cnt0==null ? new Integer(1) : new Integer(1+cnt0.intValue()); + Integer cnt = cnt0==null ? new Integer(1) : new Integer(1 + cnt0); tcnt.put(t, cnt); if (cnt==2) { tord.put(t,tord.size()); @@ -468,7 +525,7 @@ } } } - return rp.toArray(new PhrasePositions[0]); + return rp.toArray(new PhrasePositions[rp.size()]); } /** bit-sets - for each repeating pp, for each of its repeating terms, the term ordinal values is set */ @@ -508,7 +565,7 @@ /** map each term to the single group that contains it */ private HashMap termGroups(LinkedHashMap tord, ArrayList bb) throws IOException { HashMap tg = new HashMap<>(); - Term[] t = tord.keySet().toArray(new Term[0]); + Term[] t = tord.keySet().toArray(new Term[tord.size()]); for (int i=0; i= bits.length() ? DocIdSetIterator.NO_MORE_DOCS : bits.nextSetBit(ord + 1)) { @@ -518,16 +575,56 @@ return tg; } + /** + * Score a candidate doc for all slop-valid position-combinations (matches) + * encountered while traversing/hopping the PhrasePositions. + *
The score contribution of a match depends on the distance: + *
- highest score for distance=0 (exact match). + *
- score gets lower as distance gets higher. + *
Example: for query "a b"~2, a document "x a b a y" can be scored twice: + * once for "a b" (distance=0), and once for "b a" (distance=2). + *
Possibly not all valid combinations are encountered, because for efficiency + * we always propagate the least PhrasePosition. This allows to base on + * PriorityQueue and move forward faster. + * As result, for example, document "a b c b a" + * would score differently for queries "a b c"~4 and "c b a"~4, although + * they really are equivalent. + * Similarly, for doc "a b c b a f g", query "c b"~2 + * would get same score as "g f"~2, although "c b"~2 could be matched twice. + * We may want to fix this in the future (currently not, for performance reasons). + */ + float sloppyFreq() throws IOException { + if (sloppyFreq == -1) { + sloppyFreq = 0.0f; + while (nextPosition() != NO_MORE_POSITIONS) { + sloppyFreq += docScorer.computeSlopFactor(matchLength); + } + } + return sloppyFreq; + } + @Override - public int freq() { - return numMatches; + protected int doStartOffset() throws IOException { + return startoffset; } - - float sloppyFreq() { - return sloppyFreq; + + @Override + protected int doEndOffset() throws IOException { + return spanOffsetEnd; } - -// private void printQueue(PrintStream ps, PhrasePositions ext, String title) { + + // TODO : getPayload on spans? + @Override + protected int doStartPosition() throws IOException { + return startpos; + } + + @Override + protected int doEndPosition() throws IOException { + return spanEnd; + } + + // private void printQueue(PrintStream ps, PhrasePositions ext, String title) { // //if (min.doc != ?) return; // ps.println(); // ps.println("---- "+title); @@ -569,8 +666,8 @@ } @Override - public float score() { - return docScorer.score(max.doc, sloppyFreq); + public float score() throws IOException { + return docScorer.score(max.doc, sloppyFreq()); } @Override @@ -586,9 +683,9 @@ } } // found a doc with all of the terms - sloppyFreq = phraseFreq(); // check for phrase + //sloppyFreq = phraseFreq(); // check for phrase target = min.doc + 1; // next target in case sloppyFreq is still 0 - } while (sloppyFreq == 0f); + } while (firstPosition() == NO_MORE_POSITIONS); // found a match return max.doc; @@ -601,4 +698,5 @@ @Override public String toString() { return "scorer(" + weight + ")"; } + } Index: lucene/core/src/java/org/apache/lucene/search/PositionQueue.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PositionQueue.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/PositionQueue.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,133 @@ +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.posfilter.Interval; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; + +/** + * Copyright (c) 2013 Lemur Consulting Ltd. + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class PositionQueue extends PriorityQueue { + + public class DocsEnumRef { + + public final DocsEnum docsEnum; + public final int ord; + public Interval interval = new Interval(); + + public DocsEnumRef(DocsEnum docsEnum, int ord) { + this.docsEnum = docsEnum; + this.ord = ord; + } + + public int nextPosition() throws IOException { + assert docsEnum.docID() != -1; + if (docsEnum.docID() == DocsEnum.NO_MORE_DOCS || docsEnum.docID() != docId + || docsEnum.nextPosition() == DocsEnum.NO_MORE_POSITIONS) + interval.setMaximum(); + else + interval.update(this.docsEnum); + return interval.begin; + } + + } + + boolean positioned = false; + Interval current = new Interval(); + int docId = -1; + protected int queuesize; + + public PositionQueue(DocsEnum... subDocsEnums) { + super(subDocsEnums.length); + for (int i = 0; i < subDocsEnums.length; i++) { + add(new DocsEnumRef(subDocsEnums[i], i)); + } + queuesize = subDocsEnums.length; + } + + protected void init() throws IOException { + queuesize = 0; + for (Object scorerRef : getHeapArray()) { + if (scorerRef != null) { + ((DocsEnumRef) scorerRef).nextPosition(); + queuesize++; + } + } + updateTop(); + } + + public int nextPosition() throws IOException { + if (!positioned) { + init(); + positioned = true; + current.update(top().interval); + return current.begin; + } + if (current.begin == DocsEnum.NO_MORE_POSITIONS) + return DocsEnum.NO_MORE_POSITIONS; + if (top().nextPosition() == DocsEnum.NO_MORE_POSITIONS) + queuesize--; + updateInternalIntervals(); + updateTop(); + current.update(top().interval); + //System.out.println("PQ: " + current.toString()); + return current.begin; + } + + @Override + protected boolean lessThan(DocsEnumRef a, DocsEnumRef b) { + if (a.docsEnum.docID() < b.docsEnum.docID()) + return true; + if (a.docsEnum.docID() > b.docsEnum.docID()) + return false; + return a.interval.begin < b.interval.begin; + } + + protected void updateInternalIntervals() {} + + /** + * Must be called after the scorers have been advanced + */ + public void advanceTo(int doc) { + positioned = false; + this.docId = doc; + this.queuesize = this.size(); + } + + public int startPosition() throws IOException { + return current.begin; + } + + public int endPosition() throws IOException { + return current.end; + } + + public int startOffset() throws IOException { + return current.offsetBegin; + } + + public int endOffset() throws IOException { + return current.offsetEnd; + } + + public BytesRef getPayload() throws IOException { + return top().docsEnum.getPayload(); + } + + +} Property changes on: lucene/core/src/java/org/apache/lucene/search/PositionQueue.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/QueryRescorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,14 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.LeafReaderContext; + import java.io.IOException; import java.util.Arrays; import java.util.Comparator; import java.util.List; -import org.apache.lucene.index.LeafReaderContext; - /** A {@link Rescorer} that uses a provided Query to assign * scores to the first-pass hits. * @@ -82,7 +83,7 @@ if (readerContext != null) { // We advanced to another segment: docBase = readerContext.docBase; - scorer = weight.scorer(readerContext, null); + scorer = weight.scorer(readerContext, DocsEnum.FLAG_NONE, null); } if(scorer != null) { Index: lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/MinShouldMatchSumScorer.java (.../branches/lucene2878) (revision 1643331) @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; /** * A Scorer for OR like queries, counterpart of ConjunctionScorer. @@ -62,6 +63,8 @@ private final float coord[]; + private final PositionQueue posQueue; + /** * Construct a MinShouldMatchSumScorer. * @@ -110,6 +113,8 @@ this.coord = coord; minheapHeapify(); assert minheapCheck(); + + posQueue = new PositionQueue(subScorers.toArray(new Scorer[subScorers.size()])); } @Override @@ -145,6 +150,7 @@ break; } } + posQueue.advanceTo(doc); return doc; } @@ -231,6 +237,36 @@ return nrMatchers; } + @Override + public int nextPosition() throws IOException { + return posQueue.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + return posQueue.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return posQueue.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return posQueue.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return posQueue.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return posQueue.getPayload(); + } + /** * Advances to the first match beyond the current whose document number is * greater than or equal to a given target.
@@ -261,6 +297,7 @@ evaluateSmallestDocInHeap(); if (nrMatchers >= mm) { + posQueue.advanceTo(doc); return doc; } else { return nextDoc(); Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (.../branches/lucene2878) (revision 1643331) @@ -16,6 +16,11 @@ * limitations under the License. */ +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.util.Bits; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -23,11 +28,6 @@ import java.util.List; import java.util.Set; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.util.Bits; - /** * A query that generates the union of documents produced by its subqueries, and that scores each document with the maximum * score for that document as produced by any subquery, plus a tie breaking increment for any additional matching subqueries. @@ -153,11 +153,11 @@ /** Create the scorer used to score our associated DisjunctionMaxQuery */ @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { List scorers = new ArrayList<>(); for (Weight w : weights) { // we will advance() subscorers - Scorer subScorer = w.scorer(context, acceptDocs); + Scorer subScorer = w.scorer(context, flags, acceptDocs); if (subScorer != null) { scorers.add(subScorer); } Index: lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,12 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.ComplexExplanation; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.Similarity; @@ -148,7 +149,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new PayloadNearSpanScorer(query.getSpans(context, acceptDocs, termContexts), this, similarity, similarity.simScorer(stats, context)); } @@ -155,7 +156,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - PayloadNearSpanScorer scorer = (PayloadNearSpanScorer) scorer(context, context.reader().getLiveDocs()); + PayloadNearSpanScorer scorer = (PayloadNearSpanScorer) scorer(context, DocsEnum.FLAG_PAYLOADS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { Index: lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (.../branches/lucene2878) (revision 1643331) @@ -19,7 +19,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; @@ -79,7 +79,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new PayloadTermSpanScorer((TermSpans) query.getSpans(context, acceptDocs, termContexts), this, similarity.simScorer(stats, context)); } @@ -120,7 +120,7 @@ protected void processPayload(Similarity similarity) throws IOException { if (termSpans.isPayloadAvailable()) { - final DocsAndPositionsEnum postings = termSpans.getPostings(); + final DocsEnum postings = termSpans.getPostings(); payload = postings.getPayload(); if (payload != null) { payloadScore = function.currentScore(doc, term.field(), @@ -176,7 +176,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - PayloadTermSpanScorer scorer = (PayloadTermSpanScorer) scorer(context, context.reader().getLiveDocs()); + PayloadTermSpanScorer scorer = (PayloadTermSpanScorer) scorer(context, DocsEnum.FLAG_POSITIONS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { Index: lucene/core/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TermQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/TermQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,71 +17,75 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; -/** A Query that matches documents containing a term. - This may be combined with other terms with a {@link BooleanQuery}. - */ +import java.io.IOException; +import java.util.Set; + +/** + * A Query that matches documents containing a term. This may be combined with + * other terms with a {@link BooleanQuery}. + */ public class TermQuery extends Query { private final Term term; private final int docFreq; private final TermContext perReaderTermState; - + final class TermWeight extends Weight { private final Similarity similarity; private final Similarity.SimWeight stats; private final TermContext termStates; - + public TermWeight(IndexSearcher searcher, TermContext termStates) - throws IOException { + throws IOException { assert termStates != null : "TermContext must not be null"; this.termStates = termStates; this.similarity = searcher.getSimilarity(); - this.stats = similarity.computeWeight( - getBoost(), - searcher.collectionStatistics(term.field()), + this.stats = similarity.computeWeight(getBoost(), + searcher.collectionStatistics(term.field()), searcher.termStatistics(term, termStates)); } - + @Override - public String toString() { return "weight(" + TermQuery.this + ")"; } - + public String toString() { + return "weight(" + TermQuery.this + ")"; + } + @Override - public Query getQuery() { return TermQuery.this; } - + public Query getQuery() { + return TermQuery.this; + } + @Override public float getValueForNormalization() { return stats.getValueForNormalization(); } - + @Override public void normalize(float queryNorm, float topLevelBoost) { stats.normalize(queryNorm, topLevelBoost); } - + @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context); final TermsEnum termsEnum = getTermsEnum(context); if (termsEnum == null) { return null; } - DocsEnum docs = termsEnum.docs(acceptDocs, null); + DocsEnum docs = termsEnum.docs(acceptDocs, null, flags); assert docs != null; return new TermScorer(this, docs, similarity.simScorer(stats, context)); } @@ -96,8 +100,10 @@ assert termNotInReader(context.reader(), term) : "no termstate found but term exists in reader term=" + term; return null; } - //System.out.println("LD=" + reader.getLiveDocs() + " set?=" + (reader.getLiveDocs() != null ? reader.getLiveDocs().get(0) : "null")); - final TermsEnum termsEnum = context.reader().terms(term.field()).iterator(null); + // System.out.println("LD=" + reader.getLiveDocs() + " set?=" + + // (reader.getLiveDocs() != null ? reader.getLiveDocs().get(0) : "null")); + final TermsEnum termsEnum = context.reader().terms(term.field()) + .iterator(null); termsEnum.seekExact(term.bytes(), state); return termsEnum; } @@ -104,13 +110,14 @@ private boolean termNotInReader(LeafReader reader, Term term) throws IOException { // only called from assert - //System.out.println("TQ.termNotInReader reader=" + reader + " term=" + field + ":" + bytes.utf8ToString()); + // System.out.println("TQ.termNotInReader reader=" + reader + " term=" + + // field + ":" + bytes.utf8ToString()); return reader.docFreq(term) == 0; } @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context, context.reader().getLiveDocs()); + Scorer scorer = scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { @@ -117,8 +124,10 @@ float freq = scorer.freq(); SimScorer docScorer = similarity.simScorer(stats, context); ComplexExplanation result = new ComplexExplanation(); - result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:"); - Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "termFreq=" + freq)); + result.setDescription("weight(" + getQuery() + " in " + doc + ") [" + + similarity.getClass().getSimpleName() + "], result of:"); + Explanation scoreExplanation = docScorer.explain(doc, + new Explanation(freq, "termFreq=" + freq)); result.addDetail(scoreExplanation); result.setValue(scoreExplanation.getValue()); result.setMatch(true); @@ -125,18 +134,19 @@ return result; } } - return new ComplexExplanation(false, 0.0f, "no matching term"); + return new ComplexExplanation(false, 0.0f, "no matching term"); } } - + /** Constructs a query for the term t. */ public TermQuery(Term t) { this(t, -1); } - - /** Expert: constructs a TermQuery that will use the - * provided docFreq instead of looking up the docFreq - * against the searcher. */ + + /** + * Expert: constructs a TermQuery that will use the provided docFreq instead + * of looking up the docFreq against the searcher. + */ public TermQuery(Term t, int docFreq) { term = t; this.docFreq = docFreq; @@ -143,9 +153,10 @@ perReaderTermState = null; } - /** Expert: constructs a TermQuery that will use the - * provided docFreq instead of looking up the docFreq - * against the searcher. */ + /** + * Expert: constructs a TermQuery that will use the provided docFreq instead + * of looking up the docFreq against the searcher. + */ public TermQuery(Term t, TermContext states) { assert states != null; term = t; @@ -152,34 +163,37 @@ docFreq = states.docFreq(); perReaderTermState = states; } - + /** Returns the term of this query. */ - public Term getTerm() { return term; } - + public Term getTerm() { + return term; + } + @Override public Weight createWeight(IndexSearcher searcher) throws IOException { final IndexReaderContext context = searcher.getTopReaderContext(); final TermContext termState; - if (perReaderTermState == null || perReaderTermState.topReaderContext != context) { - // make TermQuery single-pass if we don't have a PRTS or if the context differs! + if (perReaderTermState == null + || perReaderTermState.topReaderContext != context) { + // make TermQuery single-pass if we don't have a PRTS or if the context + // differs! termState = TermContext.build(context, term); } else { - // PRTS was pre-build for this IS - termState = this.perReaderTermState; + // PRTS was pre-build for this IS + termState = this.perReaderTermState; } - + // we must not ignore the given docFreq - if set use the given value (lie) - if (docFreq != -1) - termState.setDocFreq(docFreq); + if (docFreq != -1) termState.setDocFreq(docFreq); return new TermWeight(searcher, termState); } - + @Override public void extractTerms(Set terms) { terms.add(getTerm()); } - + /** Prints a user-readable version of this query. */ @Override public String toString(String field) { @@ -192,21 +206,20 @@ buffer.append(ToStringUtils.boost(getBoost())); return buffer.toString(); } - + /** Returns true iff o is equal to this. */ @Override public boolean equals(Object o) { - if (!(o instanceof TermQuery)) - return false; - TermQuery other = (TermQuery)o; + if (!(o instanceof TermQuery)) return false; + TermQuery other = (TermQuery) o; return (this.getBoost() == other.getBoost()) - && this.term.equals(other.term); + && this.term.equals(other.term); } - - /** Returns a hash code value for this object.*/ + + /** Returns a hash code value for this object. */ @Override public int hashCode() { return Float.floatToIntBits(getBoost()) ^ term.hashCode(); } - + } Index: lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,12 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.BooleanQuery.BooleanWeight; + import java.io.IOException; import java.util.List; -import org.apache.lucene.search.BooleanQuery.BooleanWeight; - /* Description from Doug Cutting (excerpted from * LUCENE-1483): * @@ -98,8 +99,13 @@ return true; } + @Override + public int postingFeatures() { + return DocsEnum.FLAG_FREQS; + } + } - + static final class Bucket { int doc = -1; // tells if bucket is valid double score; // incremental score Index: lucene/core/src/java/org/apache/lucene/search/CachingCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,16 @@ * limitations under the License. */ -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.RamUsageEstimator; - import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; + /** * Caches all docs, and optionally also scores, coming from * a search, and is then able to replay them to another @@ -74,10 +75,41 @@ public final int freq() { throw new UnsupportedOperationException(); } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public final int nextDoc() { throw new UnsupportedOperationException(); } @Override public long cost() { return 1; } + } private static class NoScoreCachingCollector extends CachingCollector { @@ -96,6 +128,11 @@ docs = new ArrayList<>(); } + @Override + public int postingFeatures() { + return in.postingFeatures(); + } + protected NoScoreCachingLeafCollector wrap(LeafCollector in, int maxDocsToCache) { return new NoScoreCachingLeafCollector(in, maxDocsToCache); } @@ -304,7 +341,7 @@ * @param acceptDocsOutOfOrder * whether documents are allowed to be collected out-of-order */ - public static CachingCollector create(final boolean acceptDocsOutOfOrder, boolean cacheScores, double maxRAMMB) { + public static CachingCollector create(final boolean acceptDocsOutOfOrder, final int flags, boolean cacheScores, double maxRAMMB) { Collector other = new SimpleCollector() { @Override public boolean acceptsDocsOutOfOrder() { @@ -312,6 +349,11 @@ } @Override + public int postingFeatures() { + return flags; + } + + @Override public void collect(int doc) {} }; Index: lucene/core/src/java/org/apache/lucene/search/package.html =================================================================== --- lucene/core/src/java/org/apache/lucene/search/package.html (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/package.html (.../branches/lucene2878) (revision 1643331) @@ -436,15 +436,15 @@ that scores via a {@link org.apache.lucene.search.similarities.Similarity Similarity} will just defer to the Similarity's implementation: {@link org.apache.lucene.search.similarities.Similarity.SimWeight#normalize SimWeight#normalize(float,float)}.

  • - {@link org.apache.lucene.search.Weight#scorer(org.apache.lucene.index.LeafReaderContext, org.apache.lucene.util.Bits) - scorer(LeafReaderContext context, Bits acceptDocs)} — + {@link org.apache.lucene.search.Weight#scorer(org.apache.lucene.index.LeafReaderContext, int, org.apache.lucene.util.Bits) + scorer(LeafReaderContext context, int flags, Bits acceptDocs)} — Construct a new {@link org.apache.lucene.search.Scorer Scorer} for this Weight. See The Scorer Class below for help defining a Scorer. As the name implies, the Scorer is responsible for doing the actual scoring of documents given the Query.
  • - {@link org.apache.lucene.search.Weight#bulkScorer(org.apache.lucene.index.LeafReaderContext, boolean, org.apache.lucene.util.Bits) - scorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs)} — + {@link org.apache.lucene.search.Weight#bulkScorer(org.apache.lucene.index.LeafReaderContext, boolean, int, org.apache.lucene.util.Bits) + scorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs)} — Construct a new {@link org.apache.lucene.search.BulkScorer BulkScorer} for this Weight. See The BulkScorer Class below for help defining a BulkScorer. This is an optional method, and most queries do not implement it.
    • Index: lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (.../branches/lucene2878) (revision 1643331) @@ -27,7 +27,7 @@ * This Scorer implements {@link Scorer#advance(int)}, * and it uses the skipTo() on the given scorers. */ -class ReqExclScorer extends Scorer { +class ReqExclScorer extends FilterScorer { private Scorer reqScorer; private DocIdSetIterator exclDisi; private int doc = -1; @@ -37,7 +37,7 @@ * @param exclDisi indicates exclusion. */ public ReqExclScorer(Scorer reqScorer, DocIdSetIterator exclDisi) { - super(reqScorer.weight); + super(reqScorer); this.reqScorer = reqScorer; this.exclDisi = exclDisi; } @@ -103,11 +103,6 @@ public float score() throws IOException { return reqScorer.score(); // reqScorer may be null when next() or skipTo() already return false } - - @Override - public int freq() throws IOException { - return reqScorer.freq(); - } @Override public Collection getChildren() { @@ -129,8 +124,4 @@ return doc = toNonExcluded(); } - @Override - public long cost() { - return reqScorer.cost(); - } } Index: lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (.../branches/lucene2878) (revision 1643331) @@ -17,6 +17,23 @@ * limitations under the License. */ +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.index.StoredFieldVisitor; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.ThreadInterruptedException; + import java.io.IOException; import java.util.Arrays; import java.util.Iterator; @@ -32,23 +49,6 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DirectoryReader; // javadocs -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.index.ReaderUtil; -import org.apache.lucene.index.StoredDocument; -import org.apache.lucene.index.StoredFieldVisitor; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermContext; -import org.apache.lucene.index.Terms; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.store.NIOFSDirectory; // javadoc -import org.apache.lucene.util.ThreadInterruptedException; -import org.apache.lucene.index.IndexWriter; // javadocs - /** Implements search over a single IndexReader. * *

    Applications usually need only call the inherited @@ -608,7 +608,7 @@ // continue with the following leaf continue; } - BulkScorer scorer = weight.bulkScorer(ctx, !leafCollector.acceptsDocsOutOfOrder(), ctx.reader().getLiveDocs()); + BulkScorer scorer = weight.bulkScorer(ctx, !leafCollector.acceptsDocsOutOfOrder(), collector.postingFeatures(), ctx.reader().getLiveDocs()); if (scorer != null) { try { scorer.score(leafCollector); Index: lucene/core/src/java/org/apache/lucene/search/FakeScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FakeScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/FakeScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,17 @@ * limitations under the License. */ +import java.io.IOException; import java.util.Collection; +import org.apache.lucene.util.BytesRef; + /** Used by {@link BulkScorer}s that need to pass a {@link * Scorer} to {@link LeafCollector#setScorer}. */ -final class FakeScorer extends Scorer { - float score; - int doc = -1; - int freq = 1; +public final class FakeScorer extends Scorer { + public float score; + public int doc = -1; + public int freq = 1; public FakeScorer() { super(null); @@ -46,6 +49,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support nextPosition()"); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support startPosition()"); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support endPosition()"); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support startOffset()"); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support endOffset()"); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support getPayload()"); + } + + @Override public int nextDoc() { throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()"); } Index: lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (.../branches/lucene2878) (revision 1643331) @@ -20,22 +20,25 @@ import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.index.*; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.PhraseQuery.TermDocsEnumFactory; import org.apache.lucene.search.similarities.Similarity; -final class ExactPhraseScorer extends Scorer { +final class ExactPhraseScorer extends PhraseScorer { private final int endMinus1; - + private final static int CHUNK = 4096; - + private int gen; private final int[] counts = new int[CHUNK]; private final int[] gens = new int[CHUNK]; + private final int[] offsets = new int[CHUNK]; private final long cost; private final static class ChunkState { - final DocsAndPositionsEnum posEnum; + final TermDocsEnumFactory factory; + final DocsEnum posEnum; final int offset; int posUpto; int posLimit; @@ -42,25 +45,25 @@ int pos; int lastPos; - public ChunkState(DocsAndPositionsEnum posEnum, int offset) { + public ChunkState(TermDocsEnumFactory factory, DocsEnum posEnum, int offset) { + this.factory = factory; this.posEnum = posEnum; this.offset = offset; } } - + private final ChunkState[] chunkStates; - private final DocsAndPositionsEnum lead; + private final DocsEnum lead; private int docID = -1; - private int freq; private final Similarity.SimScorer docScorer; - + ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, Similarity.SimScorer docScorer) throws IOException { super(weight); this.docScorer = docScorer; - + chunkStates = new ChunkState[postings.length]; endMinus1 = postings.length-1; @@ -70,7 +73,7 @@ cost = lead.cost(); for(int i=0;i 0) { + } else if (firstPosition() != NO_MORE_POSITIONS) { return doc; // success: matches phrase } else { doc = lead.nextDoc(); // doesn't match phrase @@ -103,7 +106,7 @@ doc = lead.advance(doc); } } - + @Override public int nextDoc() throws IOException { return docID = doNext(lead.nextDoc()); @@ -113,51 +116,114 @@ public int advance(int target) throws IOException { return docID = doNext(lead.advance(target)); } - + @Override public String toString() { return "ExactPhraseScorer(" + weight + ")"; } - + @Override - public int freq() { - return freq; - } - - @Override public int docID() { return docID; } - + @Override - public float score() { - return docScorer.score(docID, freq); + public float score() throws IOException { + return docScorer.score(docID, freq()); } - private int phraseFreq() throws IOException { + private int chunkStart = 0; + private int chunkEnd = CHUNK; - freq = 0; + private int posRemaining; + private int positionsInChunk; + private boolean cached = false; - // init chunks - for(int i=0;i cs.lastPos) { cs.lastPos = cs.pos; final int posIndex = cs.pos - chunkStart; counts[posIndex] = 1; + any = true; assert gens[posIndex] != gen; gens[posIndex] = gen; + offsets[posIndex] = cs.posEnum.startOffset(); } if (cs.posUpto == cs.posLimit) { - end = true; + exhausted = true; break; } cs.posUpto++; @@ -188,11 +258,10 @@ } // middle terms - boolean any = true; - for(int t=1;t cs.lastPos) { cs.lastPos = cs.pos; final int posIndex = cs.pos - chunkStart; @@ -204,7 +273,7 @@ } if (cs.posUpto == cs.posLimit) { - end = true; + exhausted = true; break; } cs.posUpto++; @@ -220,6 +289,8 @@ // petered out for this chunk chunkStart += CHUNK; chunkEnd += CHUNK; + if (exhausted) + return false; continue; } @@ -226,18 +297,20 @@ // last term { + any = false; final ChunkState cs = chunkStates[endMinus1]; - while(cs.pos < chunkEnd) { + while (cs.pos < chunkEnd) { if (cs.pos > cs.lastPos) { cs.lastPos = cs.pos; final int posIndex = cs.pos - chunkStart; - if (posIndex >= 0 && gens[posIndex] == gen && counts[posIndex] == endMinus1) { - freq++; + if (posIndex >= 0 && gens[posIndex] == gen + && counts[posIndex] == endMinus1) { + addPosition(cs.pos, offsets[posIndex], cs.posEnum.endOffset()); + any = true; } } if (cs.posUpto == cs.posLimit) { - end = true; break; } cs.posUpto++; @@ -247,9 +320,12 @@ chunkStart += CHUNK; chunkEnd += CHUNK; + + if (any) { + posRemaining = positionsInChunk; + return true; + } } - - return freq; } @Override @@ -256,4 +332,5 @@ public long cost() { return cost; } + } Index: lucene/core/src/java/org/apache/lucene/search/posfilter/WithinFilteredScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/WithinFilteredScorer.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/WithinFilteredScorer.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,47 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.similarities.Similarity; + +import java.io.IOException; + +public class WithinFilteredScorer extends PositionFilteredScorer { + + private final int slop; + private final PositionFilteredScorer wrappedScorer; + + public WithinFilteredScorer(PositionFilteredScorer wrappedScorer, int slop, Similarity.SimScorer simScorer) { + super(wrappedScorer, simScorer); + this.slop = slop; + this.wrappedScorer = wrappedScorer; + } + + @Override + protected int doNextPosition() throws IOException { + int position; + while ((position = wrappedScorer.nextPosition()) != NO_MORE_POSITIONS) { + if (wrappedScorer.getMatchDistance() <= slop) { + current.update(wrappedScorer); + return position; + } + } + return NO_MORE_POSITIONS; + } + +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/WithinFilteredScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/OrderedNearQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/OrderedNearQuery.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/OrderedNearQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,139 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity; + +import java.io.IOException; + +/** + * A query that matches if a set of subqueries also match, and are within + * a given distance of each other within the document. The subqueries + * must appear in the document in order. + * + * N.B. Positions must be included in the index for this query to work + * + * Implements the AND< operator as defined in "Efficient Optimally Lazy Algorithms for Minimal-Interval Semantics" + * + * @lucene.experimental + */ + +public class OrderedNearQuery extends PositionFilterQuery { + + /** + * Constructs an OrderedNearQuery + * @param slop the maximum distance between the subquery matches + * @param subqueries the subqueries to match. + */ + public OrderedNearQuery(int slop, Query... subqueries) { + super(buildBooleanQuery(subqueries), new OrderedNearScorerFactory(slop)); + } + + private static class OrderedNearScorerFactory implements ScorerFilterFactory { + + private final int slop; + + public OrderedNearScorerFactory(int slop) { + this.slop = slop; + } + + @Override + public Scorer scorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + return new WithinFilteredScorer(new OrderedNearScorer(filteredScorer, simScorer), slop, simScorer); + } + + @Override + public String getName() { + return "OrderedNear/" + slop; + } + } + + private static class OrderedNearScorer extends PositionFilteredScorer { + + private final int lastiter; + + private int index = 1; + private Interval[] intervals; + + public OrderedNearScorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + super(filteredScorer, simScorer); + intervals = new Interval[subScorers.length]; + for (int i = 0; i < subScorers.length; i++) { + intervals[i] = new Interval(); + } + lastiter = intervals.length - 1; + } + + @Override + public int freq() throws IOException { + return 1; // nocommit + } + + @Override + protected void reset(int doc) throws IOException { + for (int i = 0; i < subScorers.length; i++) { + assert subScorers[i].docID() == doc; + intervals[i].update(Interval.INFINITE_INTERVAL); + } + if (subScorers[0].nextPosition() == NO_MORE_POSITIONS) + intervals[0].setMaximum(); + else + intervals[0].update(subScorers[0]); + index = 1; + } + + @Override + protected int doNextPosition() throws IOException { + if (intervals[0].begin == NO_MORE_POSITIONS) + return NO_MORE_POSITIONS; + current.setMaximum(); + int b = Integer.MAX_VALUE; + while (true) { + while (true) { + final Interval previous = intervals[index - 1]; + if (previous.end >= b) { + return current.begin; + } + if (index == intervals.length || intervals[index].begin > previous.end) + break; + Interval scratch = intervals[index]; + do { + if (scratch.end >= b || subScorers[index].nextPosition() == NO_MORE_POSITIONS) + return current.begin; + intervals[index].update(subScorers[index]); + scratch = intervals[index]; + } while (scratch.begin <= previous.end); + index++; + } + current.update(intervals[0], intervals[lastiter]); + matchDistance = (intervals[lastiter].begin - lastiter) - intervals[0].end; + b = intervals[lastiter].begin; + index = 1; + if (subScorers[0].nextPosition() == NO_MORE_POSITIONS) { + intervals[0].setMaximum(); + return current.begin; + } + intervals[0].update(subScorers[0]); + } + } + } +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/OrderedNearQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilterQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilterQuery.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilterQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,164 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermContext; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.Bits; + +import java.io.IOException; +import java.util.Set; +import java.util.TreeSet; + +public class PositionFilterQuery extends Query { + + protected final Query innerQuery; + protected final ScorerFilterFactory scorerFilterFactory; + + public PositionFilterQuery(Query innerQuery, ScorerFilterFactory scorerFilterFactory) { + this.innerQuery = innerQuery; + this.scorerFilterFactory = scorerFilterFactory; + } + + protected static BooleanQuery buildBooleanQuery(Query... queries) { + BooleanQuery bq = new BooleanQuery(); + for (Query q : queries) { + bq.add(q, BooleanClause.Occur.MUST); + } + return bq; + } + + @Override + public void extractTerms(Set terms) { + innerQuery.extractTerms(terms); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = innerQuery.rewrite(reader); + if (rewritten != innerQuery) { + return new PositionFilterQuery(rewritten, scorerFilterFactory); + } + return this; + } + + @Override + public Weight createWeight(IndexSearcher searcher) throws IOException { + return new ScorerFilterWeight(innerQuery.createWeight(searcher), searcher); + } + + @Override + public String toString(String field) { + return scorerFilterFactory.getName() + "[" + innerQuery.toString() + "]"; + } + + public class ScorerFilterWeight extends Weight { + + protected final Weight innerWeight; + protected final Similarity similarity; + protected final Similarity.SimWeight stats; + + public ScorerFilterWeight(Weight innerWeight, IndexSearcher searcher) throws IOException { + this.innerWeight = innerWeight; + this.similarity = searcher.getSimilarity(); + this.stats = getSimWeight(innerWeight.getQuery(), searcher); + } + + private Similarity.SimWeight getSimWeight(Query query, IndexSearcher searcher) throws IOException { + TreeSet terms = new TreeSet(); + query.extractTerms(terms); + if (terms.size() == 0) + return null; + int i = 0; + TermStatistics[] termStats = new TermStatistics[terms.size()]; + for (Term term : terms) { + TermContext state = TermContext.build(searcher.getTopReaderContext(), term); + termStats[i] = searcher.termStatistics(term, state); + i++; + } + final String field = terms.first().field(); // nocommit - should we be checking all filtered terms + // are on the same field? + return similarity.computeWeight(query.getBoost(), searcher.collectionStatistics(field), termStats); + } + + @Override + public Explanation explain(LeafReaderContext context, int doc) throws IOException { + Scorer scorer = scorer(context, DocsEnum.FLAG_POSITIONS, context.reader().getLiveDocs()); + if (scorer != null) { + int newDoc = scorer.advance(doc); + if (newDoc == doc) { + float freq = scorer.freq(); + Similarity.SimScorer docScorer = similarity.simScorer(stats, context); + ComplexExplanation result = new ComplexExplanation(); + result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:"); + Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq)); + result.addDetail(scoreExplanation); + result.setValue(scoreExplanation.getValue()); + result.setMatch(true); + return result; + } + } + return new ComplexExplanation(false, 0.0f, + "No matching term within position filter"); + } + + @Override + public Query getQuery() { + return PositionFilterQuery.this; + } + + @Override + public float getValueForNormalization() throws IOException { + return stats == null ? 1.0f : stats.getValueForNormalization(); + } + + @Override + public void normalize(float norm, float topLevelBoost) { + if (stats != null) + stats.normalize(norm, topLevelBoost); + } + + @Override + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + Scorer filteredScorer = innerWeight.scorer(context, flags | DocsEnum.FLAG_POSITIONS, acceptDocs); + return filteredScorer == null ? null + : scorerFilterFactory.scorer(filteredScorer, similarity.simScorer(stats, context)); + } + } + + public static interface ScorerFilterFactory { + + public Scorer scorer(Scorer filteredScorer, Similarity.SimScorer simScorer); + + public String getName(); + + } +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilterQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilteredScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilteredScorer.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilteredScorer.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,143 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.BytesRef; + +public abstract class PositionFilteredScorer extends Scorer { + + protected final Scorer[] subScorers; + protected final Scorer child; + protected final Interval current = new Interval(); + protected final Similarity.SimScorer simScorer; + protected int matchDistance; + + private boolean buffered; + + public PositionFilteredScorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + super(filteredScorer.getWeight()); + this.simScorer = simScorer; + child = filteredScorer; + subScorers = new Scorer[filteredScorer.getChildren().size()]; + int i = 0; + for (ChildScorer subScorer : filteredScorer.getChildren()) { + subScorers[i++] = subScorer.child; + } + } + + @Override + public float score() throws IOException { + return this.simScorer.score(docID(), freq()); + } + + @Override + public int docID() { + return child.docID(); + } + + @Override + public int freq() throws IOException { + int freq = 0; + while (nextPosition() != NO_MORE_POSITIONS) { + freq++; + } + return freq; + } + + @Override + public int nextDoc() throws IOException { + while (child.nextDoc() != NO_MORE_DOCS) { + reset(child.docID()); + if (nextPosition() != NO_MORE_POSITIONS) { + buffered = true; + return child.docID(); + } + } + return NO_MORE_DOCS; + } + + @Override + public int advance(int target) throws IOException { + if (child.advance(target) == NO_MORE_DOCS) + return NO_MORE_DOCS; + do { + reset(child.docID()); + if (nextPosition() != NO_MORE_POSITIONS) { + buffered = true; + return child.docID(); + } + } while (child.nextDoc() != NO_MORE_DOCS); + return NO_MORE_DOCS; + } + + @Override + public int nextPosition() throws IOException { + if (buffered) { + //System.out.println(this.hashCode() + ": returning buffered nextPos"); + buffered = false; + return current.begin; + } + //System.out.println(this.hashCode() + ": returning unbuffered nextPos"); + return doNextPosition(); + } + + protected abstract int doNextPosition() throws IOException; + + protected void reset(int doc) throws IOException { + buffered = false; + }; + + public int getMatchDistance() { + return matchDistance; + } + + @Override + public int startPosition() throws IOException { + return current.begin; + } + + @Override + public int endPosition() throws IOException { + return current.end; + } + + @Override + public int startOffset() throws IOException { + return current.offsetBegin; + } + + @Override + public int endOffset() throws IOException { + return current.offsetEnd; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; // nocommit + } + + @Override + public long cost() { + return child.cost(); + } +// nocommit Payloads - need to add these to Interval? +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/PositionFilteredScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/UnorderedNearQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/UnorderedNearQuery.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/UnorderedNearQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,187 @@ +package org.apache.lucene.search.posfilter; + +/** + * Copyright (c) 2012 Lemur Consulting Ltd. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.PositionQueue; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity; + +import java.io.IOException; + +/** + * A query that matches if a set of subqueries also match, and are within + * a given distance of each other within the document. The subqueries + * may appear in the document in any order. + * + * N.B. Positions must be included in the index for this query to work + * + * Implements the LOWPASSk operator as defined in "Efficient Optimally Lazy Algorithms for Minimal-Interval Semantics" + * + * @lucene.experimental + */ + +public class UnorderedNearQuery extends PositionFilterQuery { + + /** + * Constructs an OrderedNearQuery + * @param slop the maximum distance between the subquery matches + * @param subqueries the subqueries to match. + */ + public UnorderedNearQuery(int slop, Query... subqueries) { + super(buildBooleanQuery(subqueries), new UnorderedNearScorerFactory(slop)); + } + + private static class UnorderedNearScorerFactory implements ScorerFilterFactory { + + private final int slop; + + UnorderedNearScorerFactory(int slop) { + this.slop = slop; + } + + @Override + public Scorer scorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + return new WithinFilteredScorer(new UnorderedNearScorer(filteredScorer, simScorer), slop, simScorer); + } + + @Override + public String getName() { + return "UnorderedNear/" + slop; + } + } + + private static class UnorderedNearScorer extends PositionFilteredScorer { + + SpanningPositionQueue posQueue; + + public UnorderedNearScorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + super(filteredScorer, simScorer); + posQueue = new SpanningPositionQueue(subScorers); + } + + @Override + protected int doNextPosition() throws IOException { + while (posQueue.isFull() && posQueue.span.begin == current.begin) { + posQueue.nextPosition(); + } + if (!posQueue.isFull()) + return NO_MORE_POSITIONS; + do { + //current.update(posQueue.top().interval, posQueue.span); + posQueue.updateCurrent(current); + if (current.equals(posQueue.top().interval)) + return current.begin; + matchDistance = posQueue.getMatchDistance(); + posQueue.nextPosition(); + } while (posQueue.isFull() && current.end == posQueue.span.end); + return current.begin; + } + + @Override + protected void reset(int doc) throws IOException { + super.reset(doc); + current.reset(); + posQueue.advanceTo(doc); + } + + } + + private static class SpanningPositionQueue extends PositionQueue { + + Interval span = new Interval(); + int scorerCount; + int firstIntervalEnd; + int lastIntervalBegin; + + public SpanningPositionQueue(Scorer[] subScorers) { + super(subScorers); + scorerCount = subScorers.length; + } + + public int getMatchDistance() { + return lastIntervalBegin - firstIntervalEnd - scorerCount + 1; + } + + public boolean isFull() { + return queuesize == scorerCount; + } + + public void updateCurrent(Interval current) { + final Interval top = this.top().interval; + current.update(top, span); + this.firstIntervalEnd = top.end; + } + + private void updateRightExtreme(Interval newRight) { + if (span.end <= newRight.end) { + span.update(span, newRight); + this.lastIntervalBegin = newRight.begin; + } + } + + protected void updateInternalIntervals() { + updateRightExtreme(top().interval); + } + + @Override + public int nextPosition() throws IOException { + int position; + if ((position = super.nextPosition()) == DocsEnum.NO_MORE_POSITIONS) { + return DocsEnum.NO_MORE_POSITIONS; + } + span.update(top().interval, span); + return position; + } + + @Override + protected void init() throws IOException { + super.init(); + for (Object docsEnumRef : getHeapArray()) { + if (docsEnumRef != null) { + final Interval i = ((DocsEnumRef) docsEnumRef).interval; + updateRightExtreme(i); + } + } + } + + @Override + public void advanceTo(int doc) { + super.advanceTo(doc); + span.reset(); + firstIntervalEnd = lastIntervalBegin = span.begin; + } + + @Override + protected boolean lessThan(DocsEnumRef left, DocsEnumRef right) { + final Interval a = left.interval; + final Interval b = right.interval; + return a.begin < b.begin || (a.begin == b.begin && a.end > b.end); + } + + @Override + public String toString() { + return top().interval.toString(); + } + } + + +} + Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/UnorderedNearQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/Interval.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/Interval.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/Interval.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,200 @@ +package org.apache.lucene.search.posfilter; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.DocsEnum; + +import java.io.IOException; + +/** + * Represents a section of a document that matches a query + */ +public class Interval implements Cloneable { + + /** The position of the start of this Interval */ + public int begin; + + /** The position of the end of this Interval */ + public int end; + + /** The offset of the start of this Interval */ + public int offsetBegin; + + /** The offset of the end of this Interval */ + public int offsetEnd; + + /** An interval that will always compare as less than any other interval */ + public static final Interval INFINITE_INTERVAL = new Interval(); + + /** + * Constructs a new Interval + * @param begin the start position + * @param end the end position + * @param offsetBegin the start offset + * @param offsetEnd the end offset + */ + public Interval(int begin, int end, int offsetBegin, int offsetEnd) { + this.begin = begin; + this.end = end; + this.offsetBegin = offsetBegin; + this.offsetEnd = offsetEnd; + } + + /** + * Constructs a new Interval with no initial values. This + * will always compare as less than any other Interval. + */ + public Interval() { + this(Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); + } + + public Interval(DocsEnum docsEnum) throws IOException { + this.begin = docsEnum.startPosition(); + this.end = docsEnum.endPosition(); + this.offsetBegin = docsEnum.startOffset(); + this.offsetEnd = docsEnum.endOffset(); + } + + /** + * Update to span the range defined by two other Intervals. + * @param start the first Interval + * @param end the second Interval + */ + public void update(Interval start, Interval end) { + this.begin = start.begin; + this.offsetBegin = start.offsetBegin; + this.end = end.end; + this.offsetEnd = end.offsetEnd; + } + + /** + * Compare with another Interval. + * @param other the comparator + * @return true if both start and end positions are less than + * the comparator. + */ + public boolean lessThanExclusive(Interval other) { + return begin < other.begin && end < other.end; + } + + /** + * Compare with another Interval. + * @param other the comparator + * @return true if both start and end positions are less than + * or equal to the comparator's. + */ + public boolean lessThan(Interval other) { + return begin <= other.begin && end <= other.end; + } + + /** + * Compare with another Interval + * @param other the comparator + * @return true if both start and end positions are greater then + * the comparator's. + */ + public boolean greaterThanExclusive(Interval other) { + return begin > other.begin && end > other.end; + } + + /** + * Compare with another Interval + * @param other the comparator + * @return true if both start and end positions are greater then + * of equal to the comparator's. + */ + public boolean greaterThan(Interval other) { + return begin >= other.begin && end >= other.end; + } + + /** + * Compare with another Interval + * @param other the comparator + * @return true if this Interval contains the comparator + */ + public boolean contains(Interval other) { + return begin <= other.begin && other.end <= end; + } + + /** + * Compare with another Interval to find overlaps + * @param other + * @return true if the two intervals overlap + */ + public boolean overlaps(Interval other) { + return this.contains(other) || other.contains(this); + } + + /** + * Set all values of this Interval to be equal to another's + * @param other the Interval to copy + */ + public void copy(Interval other) { + begin = other.begin; + end = other.end; + offsetBegin = other.offsetBegin; + offsetEnd = other.offsetEnd; + } + + /** + * Set to a state that will always compare as less than any + * other Interval. + */ + public void reset() { + offsetBegin = offsetEnd = -1; + begin = end = Integer.MIN_VALUE; + } + + /** + * Set to a state that will always compare as more than any + * other Interval. + */ + public void setMaximum() { + offsetBegin = offsetEnd = -1; + begin = end = Integer.MAX_VALUE; + } + + @Override + public Object clone() { + try { + return super.clone(); + } catch (CloneNotSupportedException e) { + throw new RuntimeException(); // should not happen + } + } + + @Override + public String toString() { + return "Interval [begin=" + begin + "(" + offsetBegin + "), end=" + + end + "(" + offsetEnd + ")]"; + } + + public void update(DocsEnum docsEnum) throws IOException { + offsetBegin = docsEnum.startOffset(); + offsetEnd = docsEnum.endOffset(); + begin = docsEnum.startPosition(); + end = docsEnum.endPosition(); + } + + public void update(Interval interval) { + this.begin = interval.begin; + this.end = interval.end; + this.offsetBegin = interval.offsetBegin; + this.offsetEnd = interval.offsetEnd; + } + +} \ No newline at end of file Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/Interval.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/NonOverlappingQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/NonOverlappingQuery.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/NonOverlappingQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,204 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.Bits; + +/** + * A Query that matches documents containing an interval (the minuend) that + * does not contain another interval (the subtrahend). + * + * As an example, given the following {@link org.apache.lucene.search.BooleanQuery}: + * 

    + *   BooleanQuery bq = new BooleanQuery();
+ *   bq.add(new TermQuery(new Term(field, "quick")), BooleanQuery.Occur.MUST);
+ *   bq.add(new TermQuery(new Term(field, "fox")), BooleanQuery.Occur.MUST);
+ *
    + * + * The document "the quick brown fox" will be matched by this query. But + * create a NonOverlappingQuery using this query as a minuend: + * 
    + *   NonOverlappingQuery brq = new NonOverlappingQuery(bq, new TermQuery(new Term(field, "brown")));
+ *
    + * + * This query will not match "the quick brown fox", because "brown" is found + * within the interval of the boolean query for "quick" and "fox. The query + * will match "the quick fox is brown", because here "brown" is outside + * the minuend's interval. + * + * N.B. Positions must be included in the index for this query to work + * + * Implements the Brouwerian operator as defined in "Efficient Optimally Lazy Algorithms for Minimal-Interval Semantics" + * + * @lucene.experimental + */ +public final class NonOverlappingQuery extends PositionFilterQuery { + + private Query subtrahend; + + /** + * Constructs a Query that matches documents containing intervals of the minuend + * that are not subtended by the subtrahend + * @param minuend the minuend Query + * @param subtrahend the subtrahend Query + */ + public NonOverlappingQuery(Query minuend, Query subtrahend) { + super(minuend, new BrouwerianScorerFactory(subtrahend)); + this.subtrahend = subtrahend; + } + + @Override + public void extractTerms(Set terms) { + super.extractTerms(terms); + subtrahend.extractTerms(terms); + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query rewrittenMinuend = innerQuery.rewrite(reader); + Query rewrittenSubtrahend = subtrahend.rewrite(reader); + if (rewrittenMinuend != innerQuery || rewrittenSubtrahend != subtrahend) { + return new NonOverlappingQuery(rewrittenMinuend, rewrittenSubtrahend); + } + return this; + } + + private static class BrouwerianScorerFactory implements ScorerFilterFactory { + + private final Query subtrahend; + + BrouwerianScorerFactory(Query subtrahend) { + this.subtrahend = subtrahend; + } + + @Override + public Scorer scorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return "NonOverlapping[" + subtrahend.toString() + "]/"; + } + } + + @Override + public Weight createWeight(IndexSearcher searcher) throws IOException { + return new BrouwerianWeight(innerQuery.createWeight(searcher), + subtrahend.createWeight(searcher), searcher); + } + + class BrouwerianWeight extends ScorerFilterWeight { + + private final Weight subtrahendWeight; + + public BrouwerianWeight(Weight minuendWeight, Weight subtrahendWeight, IndexSearcher searcher) + throws IOException { + super(minuendWeight, searcher); + this.subtrahendWeight = subtrahendWeight; + } + + @Override + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + return new BrouwerianScorer(innerWeight.scorer(context, flags, acceptDocs), + subtrahendWeight.scorer(context, flags, acceptDocs), + similarity.simScorer(stats, context)); + } + } + + static class BrouwerianScorer extends PositionFilteredScorer { + + private final Scorer subtrahend; + private Interval subtInterval = new Interval(); + private int subtPosition = -1; + + BrouwerianScorer(Scorer minuend, Scorer subtrahend, Similarity.SimScorer simScorer) { + super(minuend, simScorer); + this.subtrahend = subtrahend; + } + + @Override + protected void reset(int doc) throws IOException { + super.reset(doc); + if (this.subtrahend == null || this.subtrahend.docID() == NO_MORE_DOCS || this.subtrahend.advance(doc) != doc) + subtPosition = NO_MORE_POSITIONS; + else + subtPosition = -1; + this.subtInterval.reset(); + } + + @Override + protected int doNextPosition() throws IOException { + if (subtPosition == NO_MORE_POSITIONS) { + int pos = child.nextPosition(); + if (pos != NO_MORE_POSITIONS) + current.update(child); + return pos; + } + while (child.nextPosition() != NO_MORE_POSITIONS) { + current.update(child); + while (subtInterval.lessThanExclusive(current) && + (subtPosition = subtrahend.nextPosition()) != NO_MORE_POSITIONS) { + subtInterval.update(subtrahend); + } + if (subtPosition == NO_MORE_POSITIONS || !current.overlaps(subtInterval)) + return current.begin; + } + return NO_MORE_POSITIONS; + } + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((innerQuery == null) ? 0 : innerQuery.hashCode()); + result = prime * result + + ((subtrahend == null) ? 0 : subtrahend.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (!super.equals(obj)) return false; + if (getClass() != obj.getClass()) return false; + NonOverlappingQuery other = (NonOverlappingQuery) obj; + if (innerQuery == null) { + if (other.innerQuery != null) return false; + } else if (!innerQuery.equals(other.innerQuery)) return false; + if (subtrahend == null) { + if (other.subtrahend != null) return false; + } else if (!subtrahend.equals(other.subtrahend)) return false; + return true; + } + +} \ No newline at end of file Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/NonOverlappingQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/RangeFilterQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/RangeFilterQuery.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/RangeFilterQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,82 @@ +package org.apache.lucene.search.posfilter; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity; + +import java.io.IOException; + +/** + * Copyright (c) 2012 Lemur Consulting Ltd. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class RangeFilterQuery extends PositionFilterQuery { + + public RangeFilterQuery(int start, int end, Query innerQuery) { + super(innerQuery, new RangeFilterScorerFactory(start, end)); + } + + public RangeFilterQuery(int end, Query innerQuery) { + this(0, end, innerQuery); + } + + private static class RangeFilterScorerFactory implements ScorerFilterFactory { + + private final int start; + private final int end; + + public RangeFilterScorerFactory(int start, int end) { + this.start = start; + this.end = end; + } + + @Override + public Scorer scorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + return new RangeFilterScorer(start, end, filteredScorer, simScorer); + } + + @Override + public String getName() { + return "RangeFilter(" + start + "," + end + ")"; + } + } + + private static class RangeFilterScorer extends PositionFilteredScorer { + + private final int start; + private final int end; + + public RangeFilterScorer(int start, int end, Scorer filteredScorer, Similarity.SimScorer simScorer) { + super(filteredScorer, simScorer); + this.start = start; + this.end = end; + } + + @Override + protected int doNextPosition() throws IOException { + int position; + while ((position = child.nextPosition()) != NO_MORE_POSITIONS) { + if (position > end) + return NO_MORE_POSITIONS; + if (position >= start) { + current.update(child); + return position; + } + } + return NO_MORE_POSITIONS; + } + } + +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/RangeFilterQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/posfilter/BlockPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/posfilter/BlockPhraseScorer.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/posfilter/BlockPhraseScorer.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,70 @@ +package org.apache.lucene.search.posfilter; + +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.similarities.Similarity; + +import java.io.IOException; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class BlockPhraseScorer extends PositionFilteredScorer { + + private final Interval[] subIntervals; + + public BlockPhraseScorer(Scorer filteredScorer, Similarity.SimScorer simScorer) { + super(filteredScorer, simScorer); + subIntervals = new Interval[subScorers.length]; + for (int i = 0; i < subScorers.length; i++) { + subIntervals[i] = new Interval(); + } + } + + @Override + public void reset(int doc) throws IOException { + super.reset(doc); + for (int i = 0; i < subScorers.length; i++) { + subIntervals[i].reset(); + } + } + + @Override + protected int doNextPosition() throws IOException { + if (subScorers[0].nextPosition() == NO_MORE_POSITIONS) + return NO_MORE_POSITIONS; + subIntervals[0].update(subScorers[0]); + int i = 1; + while (i < subScorers.length) { + while (subIntervals[i].begin <= subIntervals[i - 1].end) { + if (subScorers[i].nextPosition() == NO_MORE_POSITIONS) + return NO_MORE_POSITIONS; + subIntervals[i].update(subScorers[i]); + } + if (subIntervals[i].begin == subIntervals[i - 1].end + 1) { + i++; + } + else { + if (subScorers[0].nextPosition() == NO_MORE_POSITIONS) + return NO_MORE_POSITIONS; + subIntervals[0].update(subScorers[0]); + i = 1; + } + } + current.update(subIntervals[0], subIntervals[subScorers.length - 1]); + return subScorers[0].startPosition(); + } +} Property changes on: lucene/core/src/java/org/apache/lucene/search/posfilter/BlockPhraseScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/PhrasePositions.java (.../branches/lucene2878) (revision 1643331) @@ -18,26 +18,28 @@ */ import java.io.IOException; -import org.apache.lucene.index.*; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Term; + /** - * Position of a term in a document that takes into account the term offset within the phrase. + * Position of a term in a document that takes into account the term phraseOffset within the phrase. */ final class PhrasePositions { int doc; // current doc int position; // position in doc - int count; // remaining pos in this doc - int offset; // position in phrase + //int count; // remaining pos in this doc + int phraseOffset; // position in phrase final int ord; // unique across all PhrasePositions instances - final DocsAndPositionsEnum postings; // stream of docs & positions + final DocsEnum postings; // stream of docs & positions PhrasePositions next; // used to make lists int rptGroup = -1; // >=0 indicates that this is a repeating PP int rptInd; // index in the rptGroup final Term[] terms; // for repetitions initialization - PhrasePositions(DocsAndPositionsEnum postings, int o, int ord, Term[] terms) { + PhrasePositions(DocsEnum postings, int o, int ord, Term[] terms) { this.postings = postings; - offset = o; + phraseOffset = o; this.ord = ord; this.terms = terms; } @@ -44,6 +46,7 @@ final boolean next() throws IOException { // increments to next doc doc = postings.nextDoc(); + if (doc == DocIdSetIterator.NO_MORE_DOCS) { return false; } @@ -59,31 +62,44 @@ } final void firstPosition() throws IOException { - count = postings.freq(); // read first pos nextPosition(); } /** * Go to next location of this term current document, and set - * position as location - offset, so that a + * position as location - phraseOffset, so that a * matching exact phrase is easily identified when all PhrasePositions * have exactly the same position. */ final boolean nextPosition() throws IOException { - if (count-- > 0) { // read subsequent pos's - position = postings.nextPosition() - offset; - return true; - } else + int nextPos = postings.nextPosition(); + if (nextPos == DocsEnum.NO_MORE_POSITIONS) { + position = nextPos; return false; + } + position = nextPos - phraseOffset; + return true; } + + public final int startOffset() throws IOException { + return postings.startOffset(); + } + + public final int endOffset() throws IOException { + return postings.endOffset(); + } /** for debug purposes */ @Override public String toString() { - String s = "d:"+doc+" o:"+offset+" p:"+position+" c:"+count; + String s = "d:"+doc+" phraseOffset:"+ phraseOffset +" position:"+position; if (rptGroup >=0 ) { s += " rpt:"+rptGroup+",i"+rptInd; } + s += " t: [" + terms[0]; + for (int i = 1; i < terms.length; i++) + s += "," + terms[1]; + s += "]"; return s; } } Index: lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/BooleanTopLevelScorers.java (.../branches/lucene2878) (revision 1643331) @@ -21,8 +21,6 @@ import java.util.Collection; import java.util.Collections; -import org.apache.lucene.search.Scorer.ChildScorer; - /** Internal document-at-a-time scorers used to deal with stupid coord() computation */ class BooleanTopLevelScorers { @@ -61,7 +59,7 @@ private final Scorer req; private final Scorer opt; - CoordinatingConjunctionScorer(Weight weight, float coords[], Scorer req, int reqCount, Scorer opt) { + CoordinatingConjunctionScorer(Weight weight, float coords[], Scorer req, int reqCount, Scorer opt) throws IOException { super(weight, new Scorer[] { req, opt }); this.coords = coords; this.req = req; Index: lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -1,62 +0,0 @@ -package org.apache.lucene.index; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.Bits; // javadocs -import org.apache.lucene.util.BytesRef; - -/** Also iterates through positions. */ -public abstract class DocsAndPositionsEnum extends DocsEnum { - - /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} - * if you require offsets in the returned enum. */ - public static final int FLAG_OFFSETS = 0x1; - - /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} - * if you require payloads in the returned enum. */ - public static final int FLAG_PAYLOADS = 0x2; - - /** Sole constructor. (For invocation by subclass - * constructors, typically implicit.) */ - protected DocsAndPositionsEnum() { - } - - /** Returns the next position. You should only call this - * up to {@link DocsEnum#freq()} times else - * the behavior is not defined. If positions were not - * indexed this will return -1; this only happens if - * offsets were indexed and you passed needsOffset=true - * when pulling the enum. */ - public abstract int nextPosition() throws IOException; - - /** Returns start offset for the current position, or -1 - * if offsets were not indexed. */ - public abstract int startOffset() throws IOException; - - /** Returns end offset for the current position, or -1 if - * offsets were not indexed. */ - public abstract int endOffset() throws IOException; - - /** Returns the payload at this position, or null if no - * payload was indexed. You should not modify anything - * (neither members of the returned BytesRef nor bytes - * in the byte[]). */ - public abstract BytesRef getPayload() throws IOException; -} Index: lucene/core/src/java/org/apache/lucene/search/Scorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/Scorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/Scorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,12 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; + import java.io.IOException; import java.util.Collection; import java.util.Collections; -import org.apache.lucene.index.DocsEnum; - /** * Expert: Common scoring functionality for different types of queries. * @@ -67,6 +67,15 @@ public Weight getWeight() { return weight; } + + @Override + public String toString() { + try { + return String.format("%d:%d(%d)->%d(%d)", docID(), startPosition(), startOffset(), endPosition(), endOffset()); + } catch (IOException e) { + return String.format("Cannot retrieve position due to IOException"); + } + } /** Returns child sub-scorers * @lucene.experimental */ Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (.../branches/lucene2878) (revision 1643331) @@ -19,9 +19,10 @@ import java.io.IOException; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.BytesRef; /** * Public for extension only. @@ -96,7 +97,39 @@ public int freq() throws IOException { return numMatches; } - + + // nocommit - make SpanScorer work with positions, or just nuke it entirely? + + @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + /** Returns the intermediate "sloppy freq" adjusted for edit distance * @lucene.internal */ // only public so .payloads can see it. @@ -103,9 +136,10 @@ public float sloppyFreq() throws IOException { return freq; } - + @Override public long cost() { return spans.cost(); } + } Index: lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (.../branches/lucene2878) (revision 1643331) @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -17,28 +17,28 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Set; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ToStringUtils; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Set; + /** A Query that matches documents containing a particular sequence of terms. * A PhraseQuery is built by QueryParser for input like "new york". * @@ -138,13 +138,15 @@ } static class PostingsAndFreq implements Comparable { - final DocsAndPositionsEnum postings; + final TermDocsEnumFactory factory; + final DocsEnum postings; final int docFreq; final int position; final Term[] terms; final int nTerms; // for faster comparisons - public PostingsAndFreq(DocsAndPositionsEnum postings, int docFreq, int position, Term... terms) { + public PostingsAndFreq(DocsEnum postings, TermDocsEnumFactory factory, int docFreq, int position, Term... terms) throws IOException { + this.factory = factory; this.postings = postings; this.docFreq = docFreq; this.position = position; @@ -245,7 +247,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { assert !terms.isEmpty(); final LeafReader reader = context.reader(); final Bits liveDocs = acceptDocs; @@ -267,7 +269,7 @@ return null; } te.seekExact(t.bytes(), state); - DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, DocsEnum.FLAG_NONE); + DocsEnum postingsEnum = te.docs(liveDocs, null, DocsEnum.FLAG_POSITIONS); // PhraseQuery on a field that did not index // positions. @@ -276,7 +278,8 @@ // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.text() + ")"); } - postingsFreqs[i] = new PostingsAndFreq(postingsEnum, te.docFreq(), positions.get(i).intValue(), t); + TermDocsEnumFactory factory = new TermDocsEnumFactory(t.bytes(), state, te, flags, acceptDocs); + postingsFreqs[i] = new PostingsAndFreq(postingsEnum, factory, te.docFreq(), positions.get(i).intValue(), t); } // sort by increasing docFreq order @@ -298,7 +301,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context, context.reader().getLiveDocs()); + Scorer scorer = scorer(context, DocsEnum.FLAG_POSITIONS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { @@ -396,4 +399,33 @@ ^ positions.hashCode(); } + static class TermDocsEnumFactory { + protected final TermsEnum termsEnum; + protected final Bits liveDocs; + protected final int flags; + + private final BytesRef term; + private final TermState termState; + + TermDocsEnumFactory(TermsEnum termsEnum, int flags, Bits liveDocs) { + this(null, null, termsEnum, flags, liveDocs); + } + + TermDocsEnumFactory(BytesRef term, TermState termState, TermsEnum termsEnum, int flags, Bits liveDocs) { + this.termsEnum = termsEnum; + this.termState = termState; + this.liveDocs = liveDocs; + this.term = term; + this.flags = flags; + } + + + public DocsEnum docsAndPositionsEnum() + throws IOException { + assert term != null; + termsEnum.seekExact(term, termState); + return termsEnum.docsAndPositions(liveDocs, null, flags); + } + + } } Index: lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/QueryWrapperFilter.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,12 @@ * limitations under the License. */ -import java.io.IOException; - +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.Bits; +import java.io.IOException; + /** * Constrains search results to only match those which also match a provided * query. @@ -56,7 +57,7 @@ return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return weight.scorer(privateContext, acceptDocs); + return weight.scorer(privateContext, DocsEnum.FLAG_FREQS, acceptDocs); } @Override Index: lucene/core/src/java/org/apache/lucene/search/TermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TermScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/TermScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,12 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.index.DocsEnum; import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** Expert: A Scorer for documents matching a Term. */ final class TermScorer extends Scorer { @@ -64,6 +65,36 @@ public int nextDoc() throws IOException { return docsEnum.nextDoc(); } + + @Override + public int nextPosition() throws IOException { + return docsEnum.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + return docsEnum.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return docsEnum.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return docsEnum.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return docsEnum.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return docsEnum.getPayload(); + } @Override public float score() throws IOException { @@ -92,5 +123,16 @@ /** Returns a string representation of this TermScorer. */ @Override - public String toString() { return "scorer(" + weight + ")"; } + public String toString() { + return "scorer(" + weight + ")[" + super.toString() + "]"; + } + + // TODO: benchmark if the specialized conjunction really benefits + // from this, or if instead its from sorting by docFreq, or both + + DocsEnum getDocsEnum() { + return docsEnum; + } + + } Index: lucene/core/src/java/org/apache/lucene/search/SimpleCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/SimpleCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/SimpleCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,6 +17,8 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; + import java.io.IOException; import org.apache.lucene.index.LeafReaderContext; @@ -50,4 +52,9 @@ @Override public abstract void collect(int doc) throws IOException; + @Override + public int postingFeatures() { + return DocsEnum.FLAG_FREQS; + } + } Index: lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,11 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.index.MultiDocsEnum.EnumWithSlice; +import org.apache.lucene.util.BytesRef; -import java.io.IOException; - /** * Exposes flex API, merged from flex API of sub-segments, * remapping docIDs (this is used for segment merging). @@ -70,6 +71,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public int docID() { return doc; } Index: lucene/core/src/java/org/apache/lucene/index/LeafReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/LeafReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/LeafReader.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,10 @@ * limitations under the License. */ +import org.apache.lucene.util.Bits; + import java.io.IOException; -import org.apache.lucene.index.IndexReader.ReaderClosedListener; -import org.apache.lucene.util.Bits; - /** {@code LeafReader} is an abstract class, providing an interface for accessing an index. Search of an index is done entirely through this abstract interface, so that any subclass which implements it is searchable. IndexReaders implemented @@ -222,11 +221,11 @@ return null; } - /** Returns {@link DocsAndPositionsEnum} for the specified + /** Returns {@link DocsEnum} for the specified * term. This will return null if the * field or term does not exist or positions weren't indexed. - * @see TermsEnum#docsAndPositions(Bits, DocsAndPositionsEnum) */ - public final DocsAndPositionsEnum termPositionsEnum(Term term) throws IOException { + * @see TermsEnum#docsAndPositions(Bits, DocsEnum) */ + public final DocsEnum termPositionsEnum(Term term) throws IOException { assert term.field() != null; assert term.bytes() != null; final Terms terms = terms(term.field()); Index: lucene/core/src/java/org/apache/lucene/index/MultiFields.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiFields.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MultiFields.java (.../branches/lucene2878) (revision 1643331) @@ -146,22 +146,22 @@ return null; } - /** Returns {@link DocsAndPositionsEnum} for the specified + /** Returns {@link DocsEnum} for the specified * field and term. This will return null if the field or * term does not exist or positions were not indexed. * @see #getTermPositionsEnum(IndexReader, Bits, String, BytesRef, int) */ - public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { - return getTermPositionsEnum(r, liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS); + public static DocsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { + return getTermPositionsEnum(r, liveDocs, field, term, DocsEnum.FLAG_OFFSETS | DocsEnum.FLAG_PAYLOADS); } - /** Returns {@link DocsAndPositionsEnum} for the specified + /** Returns {@link DocsEnum} for the specified * field and term, with control over whether offsets and payloads are * required. Some codecs may be able to optimize * their implementation when offsets and/or payloads are not * required. This will return null if the field or term does not * exist or positions were not indexed. See {@link - * TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */ - public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException { + * TermsEnum#docs(Bits,DocsEnum,int)}. */ + public static DocsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); Index: lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,12 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import java.io.IOException; + /** Implements a {@link TermsEnum} wrapping a provided * {@link SortedSetDocValues}. */ @@ -114,7 +114,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { throw new UnsupportedOperationException(); } Index: lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,6 +17,15 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.ToStringUtils; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -24,14 +33,6 @@ import java.util.List; import java.util.Set; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ToStringUtils; - /** A Query that matches documents matching boolean combinations of other * queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other * BooleanQuerys. @@ -242,7 +243,7 @@ for (Iterator wIter = weights.iterator(); wIter.hasNext();) { Weight w = wIter.next(); BooleanClause c = cIter.next(); - if (w.scorer(context, context.reader().getLiveDocs()) == null) { + if (w.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()) == null) { if (c.isRequired()) { fail = true; Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")"); @@ -305,13 +306,13 @@ } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { if (scoreDocsInOrder || minNrShouldMatch > 1) { // TODO: (LUCENE-4872) in some cases BooleanScorer may be faster for minNrShouldMatch // but the same is even true of pure conjunctions... - return super.bulkScorer(context, scoreDocsInOrder, acceptDocs); + return super.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); } List prohibited = new ArrayList(); @@ -319,7 +320,7 @@ Iterator cIter = clauses.iterator(); for (Weight w : weights) { BooleanClause c = cIter.next(); - BulkScorer subScorer = w.bulkScorer(context, false, acceptDocs); + BulkScorer subScorer = w.bulkScorer(context, false, flags, acceptDocs); if (subScorer == null) { if (c.isRequired()) { return null; @@ -328,7 +329,7 @@ // TODO: there are some cases where BooleanScorer // would handle conjunctions faster than // BooleanScorer2... - return super.bulkScorer(context, scoreDocsInOrder, acceptDocs); + return super.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); } else if (c.isProhibited()) { prohibited.add(subScorer); } else { @@ -340,7 +341,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { // initially the user provided value, // but if minNrShouldMatch == optional.size(), @@ -353,7 +354,7 @@ Iterator cIter = clauses.iterator(); for (Weight w : weights) { BooleanClause c = cIter.next(); - Scorer subScorer = w.scorer(context, acceptDocs); + Scorer subScorer = w.scorer(context, flags, acceptDocs); if (subScorer == null) { if (c.isRequired()) { return null; @@ -454,8 +455,17 @@ // scorer() will return an out-of-order scorer if requested. return true; } + + @Override + public String toString() { + StringBuffer sb = new StringBuffer("BooleanWeight["); + for (Weight weight : weights) { + sb.append(weight.toString()).append(","); + } + return sb.append("]").toString(); + } - private Scorer req(List required, boolean disableCoord) { + private Scorer req(List required, boolean disableCoord) throws IOException { if (required.size() == 1) { Scorer req = required.get(0); if (!disableCoord && maxCoord > 1) { Index: lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -184,7 +184,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits bits, DocsEnum reuse, int flags) throws IOException { return tenum.docsAndPositions(bits, reuse, flags); } Index: lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/SortedDocValuesTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,12 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import java.io.IOException; + /** Implements a {@link TermsEnum} wrapping a provided * {@link SortedDocValues}. */ @@ -114,7 +114,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { throw new UnsupportedOperationException(); } Index: lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,11 @@ * limitations under the License. */ -import org.apache.lucene.util.BytesRef; - import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.util.BytesRef; + /** * Exposes flex API, merged from flex API of sub-segments. * @@ -28,13 +28,13 @@ * @lucene.experimental */ -public final class MultiDocsAndPositionsEnum extends DocsAndPositionsEnum { +public final class MultiDocsAndPositionsEnum extends DocsEnum { private final MultiTermsEnum parent; - final DocsAndPositionsEnum[] subDocsAndPositionsEnum; + final DocsEnum[] subDocsAndPositionsEnum; private final EnumWithSlice[] subs; int numSubs; int upto; - DocsAndPositionsEnum current; + DocsEnum current; int currentBase; int doc = -1; @@ -41,7 +41,7 @@ /** Sole constructor. */ public MultiDocsAndPositionsEnum(MultiTermsEnum parent, int subReaderCount) { this.parent = parent; - subDocsAndPositionsEnum = new DocsAndPositionsEnum[subReaderCount]; + subDocsAndPositionsEnum = new DocsEnum[subReaderCount]; this.subs = new EnumWithSlice[subReaderCount]; for (int i = 0; i < subs.length; i++) { subs[i] = new EnumWithSlice(); @@ -144,6 +144,16 @@ } @Override + public int startPosition() throws IOException { + return current.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return current.endPosition(); + } + + @Override public int startOffset() throws IOException { return current.startOffset(); } @@ -159,14 +169,14 @@ } // TODO: implement bulk read more efficiently than super - /** Holds a {@link DocsAndPositionsEnum} along with the + /** Holds a {@link DocsEnum} along with the * corresponding {@link ReaderSlice}. */ public final static class EnumWithSlice { EnumWithSlice() { } - /** {@link DocsAndPositionsEnum} for this sub-reader. */ - public DocsAndPositionsEnum docsAndPositionsEnum; + /** {@link DocsEnum} for this sub-reader. */ + public DocsEnum docsAndPositionsEnum; /** {@link ReaderSlice} describing how this sub-reader * fits into the composite reader. */ Index: lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsAndPositionsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsAndPositionsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MappingMultiDocsAndPositionsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,11 @@ * limitations under the License. */ +import java.io.IOException; + +import org.apache.lucene.index.MultiDocsAndPositionsEnum.EnumWithSlice; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.index.MultiDocsAndPositionsEnum.EnumWithSlice; -import java.io.IOException; - /** * Exposes flex API, merged from flex API of sub-segments, * remapping docIDs (this is used for segment merging). @@ -29,12 +29,12 @@ * @lucene.experimental */ -final class MappingMultiDocsAndPositionsEnum extends DocsAndPositionsEnum { +final class MappingMultiDocsAndPositionsEnum extends DocsEnum { private MultiDocsAndPositionsEnum.EnumWithSlice[] subs; int numSubs; int upto; MergeState.DocMap currentMap; - DocsAndPositionsEnum current; + DocsEnum current; int currentBase; int doc = -1; private MergeState mergeState; @@ -122,6 +122,16 @@ } @Override + public int startPosition() throws IOException { + return current.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return current.endPosition(); + } + + @Override public int startOffset() throws IOException { return current.startOffset(); } Index: lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,10 @@ * limitations under the License. */ -import java.io.Reader; -import java.io.StringReader; -import java.util.Arrays; -import java.util.Random; - import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.LeafReader; @@ -40,6 +35,11 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.RegExp; +import java.io.Reader; +import java.io.StringReader; +import java.util.Arrays; +import java.util.Random; + import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; public class TestMockAnalyzer extends BaseTokenStreamTestCase { @@ -321,7 +321,7 @@ final Terms terms = fields.terms("f"); final TermsEnum te = terms.iterator(null); assertEquals(new BytesRef("a"), te.next()); - final DocsAndPositionsEnum dpe = te.docsAndPositions(null, null); + final DocsEnum dpe = te.docsAndPositions(null, null); assertEquals(0, dpe.nextDoc()); assertEquals(2, dpe.freq()); assertEquals(0, dpe.nextPosition()); Index: lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestPhrasePrefixQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,20 +17,20 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.LinkedList; + +import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.MultiFields; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; -import java.io.IOException; -import java.util.LinkedList; - /** * This class tests PhrasePrefixQuery class. */ Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestBrouwerianQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestBrouwerianQuery.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestBrouwerianQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,103 @@ +package org.apache.lucene.search.posfilter; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Query; + +public class TestBrouwerianQuery extends IntervalTestBase { + + protected void addDocs(RandomIndexWriter writer) throws IOException { + { + Document doc = new Document(); + doc.add(newField( + "field", + "The quick brown fox jumps over the lazy dog", + TextField.TYPE_STORED)); + writer.addDocument(doc); + } + + { + Document doc = new Document(); + doc.add(newField( + "field", + "The quick brown duck jumps over the lazy dog with the quick brown fox jumps", + TextField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + public void testBrouwerianBooleanQuery() throws IOException { + + Query query = new OrderedNearQuery(2, makeTermQuery("the"), + makeTermQuery("quick"), makeTermQuery("jumps")); + Query sub = makeTermQuery("fox"); + NonOverlappingQuery q = new NonOverlappingQuery(query, sub); + + checkIntervals(q, searcher, new int[][]{ + { 1, 0, 4 } + }); + } + + public void testBrouwerianBooleanQueryExcludedDoesNotExist() throws IOException { + + Query query = new OrderedNearQuery(2, makeTermQuery("the"), + makeTermQuery("quick"), makeTermQuery("jumps")); + Query sub = makeTermQuery("blox"); + NonOverlappingQuery q = new NonOverlappingQuery(query, sub); + + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 4 }, + { 1, 0, 4, 10, 14 } + }); + } + + public void testBrouwerianOverlapQuery() throws IOException { + // We want to find 'jumps NOT WITHIN 2 positions of fox' + Query sub = new UnorderedNearQuery(2, makeTermQuery("jumps"), makeTermQuery("fox")); + Query query = makeTermQuery("jumps"); + NonOverlappingQuery q = new NonOverlappingQuery(query, sub); + + checkIntervals(q, searcher, new int[][]{ + { 1, 4, 4 } + }); + } + + public void testBrouwerianNonExistentOverlapQuery() throws IOException { + Query sub = new UnorderedNearQuery(2, makeTermQuery("dog"), makeTermQuery("over")); + Query query = makeTermQuery("dog"); + NonOverlappingQuery q = new NonOverlappingQuery(query, sub); + + checkIntervals(q, searcher, new int[][]{}); + } + + public void testBrouwerianExistentOverlapQuery() throws IOException { + Query sub = new UnorderedNearQuery(1, makeTermQuery("dog"), makeTermQuery("over")); + Query query = makeTermQuery("dog"); + NonOverlappingQuery q = new NonOverlappingQuery(query, sub); + + checkIntervals(q, searcher, new int[][]{ + { 0, 8, 8 }, + { 1, 8, 8 } + }); + } + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestBrouwerianQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestIntervalScoring.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestIntervalScoring.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestIntervalScoring.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,61 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.RegexpQuery; +import org.apache.lucene.search.TopDocs; + +public class TestIntervalScoring extends IntervalTestBase { + + @Override + protected void addDocs(RandomIndexWriter writer) throws IOException { + for (String content : docFields) { + Document doc = new Document(); + doc.add(newField("field", content, TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + } + } + + private String[] docFields = { + "Should we, could we, would we?", + "It should - would it?", + "It shouldn't", + "Should we, should we, should we" + }; + + public void testOrderedNearQueryScoring() throws IOException { + OrderedNearQuery q = new OrderedNearQuery(10, makeTermQuery("should"), + makeTermQuery("would")); + checkScores(q, searcher, 1, 0); + } + + public void testEmptyMultiTermQueryScoring() throws IOException { + OrderedNearQuery q = new OrderedNearQuery(10, new RegexpQuery(new Term("field", "bar.*")), + new RegexpQuery(new Term("field", "foo.*"))); + TopDocs docs = searcher.search(q, 10); + assertEquals(docs.totalHits, 0); + } + + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestIntervalScoring.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java (.../branches/lucene2878) (revision 1643331) @@ -33,7 +33,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexWriter; @@ -284,8 +283,8 @@ public void assertTermsEnum(TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean deep) throws Exception { BytesRef term; Bits randomBits = new RandomBits(MAXDOC, random().nextDouble(), random()); - DocsAndPositionsEnum leftPositions = null; - DocsAndPositionsEnum rightPositions = null; + DocsEnum leftPositions = null; + DocsEnum rightPositions = null; DocsEnum leftDocs = null; DocsEnum rightDocs = null; @@ -306,30 +305,30 @@ leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions), rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions)); // with payloads only - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsAndPositionsEnum.FLAG_PAYLOADS), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsAndPositionsEnum.FLAG_PAYLOADS)); - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsAndPositionsEnum.FLAG_PAYLOADS), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsAndPositionsEnum.FLAG_PAYLOADS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsEnum.FLAG_PAYLOADS), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsEnum.FLAG_PAYLOADS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsEnum.FLAG_PAYLOADS), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsEnum.FLAG_PAYLOADS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsAndPositionsEnum.FLAG_PAYLOADS), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsAndPositionsEnum.FLAG_PAYLOADS)); + leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsEnum.FLAG_PAYLOADS), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsEnum.FLAG_PAYLOADS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsAndPositionsEnum.FLAG_PAYLOADS), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsAndPositionsEnum.FLAG_PAYLOADS)); + leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsEnum.FLAG_PAYLOADS), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsEnum.FLAG_PAYLOADS)); // with offsets only - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsAndPositionsEnum.FLAG_OFFSETS), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsAndPositionsEnum.FLAG_OFFSETS)); - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsAndPositionsEnum.FLAG_OFFSETS), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsAndPositionsEnum.FLAG_OFFSETS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsEnum.FLAG_OFFSETS), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsEnum.FLAG_OFFSETS)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsEnum.FLAG_OFFSETS), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsEnum.FLAG_OFFSETS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsAndPositionsEnum.FLAG_OFFSETS), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsAndPositionsEnum.FLAG_OFFSETS)); + leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsEnum.FLAG_OFFSETS), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, DocsEnum.FLAG_OFFSETS)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsAndPositionsEnum.FLAG_OFFSETS), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsAndPositionsEnum.FLAG_OFFSETS)); + leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, DocsEnum.FLAG_OFFSETS), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, DocsEnum.FLAG_OFFSETS)); // with positions only assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, DocsEnum.FLAG_NONE), @@ -389,7 +388,7 @@ /** * checks docs + freqs + positions + payloads, sequentially */ - public void assertDocsAndPositionsEnum(DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception { + public void assertDocsAndPositionsEnum(DocsEnum leftDocs, DocsEnum rightDocs) throws Exception { if (leftDocs == null || rightDocs == null) { assertNull(leftDocs); assertNull(rightDocs); @@ -462,7 +461,7 @@ /** * checks advancing docs + positions */ - public void assertPositionsSkipping(int docFreq, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws Exception { + public void assertPositionsSkipping(int docFreq, DocsEnum leftDocs, DocsEnum rightDocs) throws Exception { if (leftDocs == null || rightDocs == null) { assertNull(leftDocs); assertNull(rightDocs); Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,6 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; @@ -35,6 +29,12 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + public class TestBooleanScorer extends LuceneTestCase { private static final String FIELD = "category"; @@ -207,12 +207,12 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) { throw new UnsupportedOperationException(); } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) { return new BulkScorer() { @Override Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPhraseQueryPositions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestPhraseQueryPositions.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestPhraseQueryPositions.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,191 @@ +package org.apache.lucene.search.posfilter; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; + +public class TestPhraseQueryPositions extends IntervalTestBase { + + protected void addDocs(RandomIndexWriter writer) throws IOException { + { + Document doc = new Document(); + doc.add(newField( + "field", + // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 + "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! " + // 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + "Some like it hot, some like it cold, Some like it in the pot nine days old! " + // 31 32 33 34 35 36 + + "Pease porridge hot! Pease porridge cold!", + TextField.TYPE_STORED)); + writer.addDocument(doc); + } + + { + Document doc = new Document(); + doc.add(newField( + "field", + // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 + "Pease porridge cold! Pease porridge hot! Pease porridge in the pot nine days old! " + // 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 + + "Some like it cold, some like it hot, Some like it in the pot nine days old! " + // 31 32 33 34 35 36 + + "Pease porridge cold! Pease porridge hot!", + TextField.TYPE_STORED)); + writer.addDocument(doc); + } + { + Document doc = new Document(); + doc.add(newField("sloppy", "x a x b a", TextField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + public void testOutOfOrderSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "cold!")); + query.add(new Term("field", "porridge")); + query.setSlop(2); + checkIntervals(query, searcher, new int[][]{ + {0, 3, 5, 3, 7, 5, 7, 34, 36}, + {1, 0, 2, 0, 4, 2, 4, 31, 33, 31, 35, 33, 35 } + }); + } + + public void testOverlappingOutOfOrderSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("sloppy", "x")); + query.add(new Term("sloppy", "a")); + query.add(new Term("sloppy", "a")); + query.setSlop(2); + checkIntervals(query, searcher, new int[][]{ + {2, 0, 4, 1, 4} + }); + } + + public void testSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "hot!")); + query.setSlop(1); + checkIntervals(query, searcher, new int[][]{ + {0, 0, 2, 31, 33}, + {1, 3, 5, 34, 36} + }); + } + + public void testSloppyPhraseQueryWithRepeats() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "porridge")); + query.add(new Term("field", "pease")); + query.setSlop(1); + checkIntervals(query, searcher, new int[][]{ + {0, 0, 3, 3, 6, 31, 34}, + {1, 0, 3, 3, 6, 31, 34} + }); + } + + public void testManyTermSloppyPhraseQuery() throws IOException { + PhraseQuery query = new PhraseQuery(); + query.add(new Term("field", "pease")); + query.add(new Term("field", "porridge")); + query.add(new Term("field", "pot")); + query.setSlop(2); + checkIntervals(query, searcher, new int[][]{ + {0, 6, 10}, + {1, 6, 10} + }); + } + + public void testMultiPhrases() throws IOException { + + MultiPhraseQuery q = new MultiPhraseQuery(); + q.add(new Term("field", "pease")); + q.add(new Term("field", "porridge")); + q.add(new Term[]{ new Term("field", "hot!"), new Term("field", "cold!") }); + + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2, 3, 5, 31, 33, 34, 36 }, + { 1, 0, 2, 3, 5, 31, 33, 34, 36 } + }); + } + + public void testOverlaps() throws IOException { + PhraseQuery q = new PhraseQuery(); + q.add(new Term("field", "some")); + q.add(new Term("field", "like")); + q.add(new Term("field", "it")); + q.add(new Term("field", "cold,")); + q.add(new Term("field", "some")); + q.add(new Term("field", "like")); + checkIntervals(q, searcher, new int[][]{ + {0, 18, 23}, + {1, 14, 19} + }); + } + + public void testMatching() throws IOException { + + PhraseQuery q = new PhraseQuery(); + q.add(new Term("field", "pease")); + q.add(new Term("field", "porridge")); + q.add(new Term("field", "hot!")); + + checkIntervals(q, searcher, new int[][]{ + {0, 0, 2, 31, 33}, + {1, 3, 5, 34, 36} + }); + + } + + public void testPartialMatching() throws IOException { + + PhraseQuery q = new PhraseQuery(); + q.add(new Term("field", "pease")); + q.add(new Term("field", "porridge")); + q.add(new Term("field", "hot!")); + q.add(new Term("field", "pease")); + q.add(new Term("field", "porridge")); + q.add(new Term("field", "cold!")); + + checkIntervals(q, searcher, new int[][]{ + {0, 0, 5, 31, 36}, + }); + + } + + public void testNonMatching() throws IOException { + + PhraseQuery q = new PhraseQuery(); + q.add(new Term("field", "pease")); + q.add(new Term("field", "hot!")); + + checkIntervals(q, searcher, new int[][]{}); + + } + + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPhraseQueryPositions.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestBasicIntervals.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestBasicIntervals.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestBasicIntervals.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,191 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; + +public class TestBasicIntervals extends IntervalTestBase { + + public static final String field = "field"; + + @Override + protected void addDocs(RandomIndexWriter writer) throws IOException { + for (String content : docFields) { + Document doc = new Document(); + doc.add(newField(field, content, TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + } + } + + private String[] docFields = { + "w1 w2 w3 w4 w5", //0 + "w1 w3 w2 w3",//1 + "w1 xx w2 yy w3",//2 + "w1 w3 xx w2 yy w3",//3 + "u2 u2 u1", //4 + "u2 xx u2 u1",//5 + "u2 u2 xx u1", //6 + "u2 xx u2 yy u1", //7 + "u2 xx u1 u2",//8 + "u1 u2 xx u2",//9 + "u2 u1 xx u2",//10 + "t1 t2 t1 t3 t2 t3",//11 + "v1 v2 v3",//12 + "v1 v3 v2 v3 v4",//13 + "v4 v2 v2 v4",//14 + "v3 v4 v3"};//15 + + public void testSimpleConjunction() throws IOException { + Query q = makeAndQuery(makeTermQuery("v2"), makeTermQuery("v4")); + checkIntervals(q, searcher, new int[][]{ + { 13, 2, 2, 4, 4 }, + { 14, 0, 0, 1, 1, 2, 2, 3, 3 } + }); + } + + public void testExclusion() throws IOException { + Query q = makeBooleanQuery(makeBooleanClause("v2", BooleanClause.Occur.MUST), + makeBooleanClause("v3", BooleanClause.Occur.MUST_NOT)); + checkIntervals(q, searcher, new int[][]{ + { 14, 1, 1, 2, 2 } + }); + } + + public void testOptExclusion() throws IOException { + Query q = makeBooleanQuery(makeBooleanClause("w2", BooleanClause.Occur.SHOULD), + makeBooleanClause("w3", BooleanClause.Occur.SHOULD), + makeBooleanClause("xx", BooleanClause.Occur.MUST_NOT)); + checkIntervals(q, searcher, new int[][]{ + { 0, 1, 1, 2, 2 }, + { 1, 1, 1, 2, 2, 3, 3 } + }); + } + + public void testNestedConjunctions() throws IOException { + Query q = makeAndQuery(makeTermQuery("v2"), makeOrQuery(makeTermQuery("v3"), makeTermQuery("v4"))); + checkIntervals(q, searcher, new int[][]{ + { 12, 1, 1, 2, 2 }, + { 13, 1, 1, 2, 2, 3, 3, 4, 4 }, + { 14, 0, 0, 1, 1, 2, 2, 3, 3 } + }); + } + + public void testSingleRequiredManyOptional() throws IOException { + Query q = makeBooleanQuery(makeBooleanClause("v2", BooleanClause.Occur.MUST), + makeBooleanClause("v3", BooleanClause.Occur.SHOULD), + makeBooleanClause("v4", BooleanClause.Occur.SHOULD)); + checkIntervals(q, searcher, new int[][]{ + { 12, 1, 1, 2, 2 }, + { 13, 1, 1, 2, 2, 3, 3, 4, 4 }, + { 14, 0, 0, 1, 1, 2, 2, 3, 3 } + }); + } + + public void testSimpleTerm() throws IOException { + Query q = makeTermQuery("u2"); + checkIntervals(q, searcher, new int[][]{ + { 4, 0, 0, 1, 1 }, + { 5, 0, 0, 2, 2 }, + { 6, 0, 0, 1, 1 }, + { 7, 0, 0, 2, 2 }, + { 8, 0, 0, 3, 3 }, + { 9, 1, 1, 3, 3 }, + { 10, 0, 0, 3, 3 } + }); + } + + public void testBasicDisjunction() throws IOException { + Query q = makeOrQuery(makeTermQuery("v3"), makeTermQuery("v2")); + checkIntervals(q, searcher, new int[][]{ + { 12, 1, 1, 2, 2 }, + { 13, 1, 1, 2, 2, 3, 3 }, + { 14, 1, 1, 2, 2 }, + { 15, 0, 0, 2, 2 } + }); + } + + + + public void testOrSingle() throws Exception { + Query q = makeOrQuery(makeTermQuery("w5")); + checkIntervals(q, searcher, new int[][]{ + { 0, 4, 4 } + }); + } + + public void testOrPartialMatch() throws Exception { + Query q = makeOrQuery(makeTermQuery("w5"), makeTermQuery("xx")); + checkIntervals(q, searcher, new int[][]{ + { 0, 4, 4 }, + { 2, 1, 1 }, + { 3, 2, 2 }, + { 5, 1, 1 }, + { 6, 2, 2 }, + { 7, 1, 1 }, + { 8, 1, 1 }, + { 9, 2, 2 }, + { 10, 2, 2 }, + }); + } + + public void testOrDisjunctionMatch() throws Exception { + Query q = makeOrQuery(makeTermQuery("w5"), makeTermQuery("yy")); + checkIntervals(q, searcher, new int[][]{ + { 0, 4, 4 }, + { 2, 3, 3 }, + { 3, 4, 4 }, + { 7, 3, 3 } + }); + } + + // "t1 t2 t1 t3 t2 t3" + // ----------- + // -------- + // -------- + public void testOrSingleDocument() throws Exception { + Query q = makeOrQuery(makeTermQuery("t1"), makeTermQuery("t2"), makeTermQuery("t3")); + checkIntervals(q, searcher, new int[][]{ + { 11, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5 } + }); + } + + // andnot(andnot(w1, or(w2, flurble)), or(foo, bar)) + public void testConjunctionExclusionQuery() throws IOException { + BooleanQuery andnotinner = new BooleanQuery(); + andnotinner.add(makeTermQuery("w1"), BooleanClause.Occur.MUST); + BooleanQuery andnotinneror = new BooleanQuery(); + andnotinneror.add(makeTermQuery("w2"), BooleanClause.Occur.SHOULD); + andnotinneror.add(makeTermQuery("flurble"), BooleanClause.Occur.SHOULD); + andnotinner.add(andnotinneror, BooleanClause.Occur.MUST_NOT); + BooleanQuery outer = new BooleanQuery(); + outer.add(andnotinner, BooleanClause.Occur.MUST); + BooleanQuery andnotouteror = new BooleanQuery(); + andnotouteror.add(makeTermQuery("foo"), BooleanClause.Occur.SHOULD); + andnotouteror.add(makeTermQuery("bar"), BooleanClause.Occur.SHOULD); + outer.add(andnotouteror, BooleanClause.Occur.MUST_NOT); + checkIntervals(outer, searcher, new int[][]{}); + } + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestBasicIntervals.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,15 @@ * limitations under the License. */ +import java.io.IOException; + +import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.document.Document; public class TestPositiveScoresOnlyCollector extends LuceneTestCase { @@ -41,6 +44,36 @@ return 1; } + @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + @Override public int docID() { return idx; } @Override public int nextDoc() { @@ -51,7 +84,7 @@ idx = target; return idx < scores.length ? idx : NO_MORE_DOCS; } - + @Override public long cost() { return scores.length; Index: lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,12 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.List; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.IndexWriter; @@ -43,6 +41,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.util.List; + public class TestSpans extends LuceneTestCase { private IndexSearcher searcher; private IndexReader reader; @@ -429,7 +430,7 @@ slop, ordered); - spanScorer = searcher.createNormalizedWeight(snq).scorer(ctx, ctx.reader().getLiveDocs()); + spanScorer = searcher.createNormalizedWeight(snq).scorer(ctx, DocsEnum.FLAG_POSITIONS, ctx.reader().getLiveDocs()); } finally { searcher.setSimilarity(oldSim); } Index: lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (.../branches/lucene2878) (revision 1643331) @@ -17,8 +17,6 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -36,6 +34,8 @@ import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.IOException; + public class TestTermVectors extends LuceneTestCase { private static IndexReader reader; private static Directory directory; Index: lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java (.../branches/lucene2878) (revision 1643331) @@ -25,8 +25,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; @@ -36,6 +36,7 @@ import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; @@ -443,7 +444,7 @@ } @Override - public Scorer scorer(final LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(final LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new Scorer(null) { int docID = -1; @@ -459,6 +460,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public long cost() { return 1; } Index: lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java (.../branches/lucene2878) (revision 1643331) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.util.LuceneTestCase; +import com.carrotsearch.randomizedtesting.annotations.Seed; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; @@ -32,7 +32,9 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +@Seed("2542F68A58928060") public class TestSloppyPhraseQuery extends LuceneTestCase { private static final String S_1 = "A A A"; Index: lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,10 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; @@ -33,6 +30,10 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + public class TestTermScorer extends LuceneTestCase { protected Directory directory; private static final String FIELD = "field"; @@ -78,7 +79,7 @@ Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext() instanceof LeafReaderContext); LeafReaderContext context = (LeafReaderContext)indexSearcher.getTopReaderContext(); - BulkScorer ts = weight.bulkScorer(context, true, context.reader().getLiveDocs()); + BulkScorer ts = weight.bulkScorer(context, true, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); // we have 2 documents with the term all in them, one document for all the // other values final List docs = new ArrayList<>(); @@ -140,7 +141,7 @@ Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext() instanceof LeafReaderContext); LeafReaderContext context = (LeafReaderContext) indexSearcher.getTopReaderContext(); - Scorer ts = weight.scorer(context, context.reader().getLiveDocs()); + Scorer ts = weight.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); assertTrue("next did not return a doc", ts.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertTrue("score is not correct", ts.score() == 1.6931472f); @@ -159,7 +160,7 @@ Weight weight = indexSearcher.createNormalizedWeight(termQuery); assertTrue(indexSearcher.getTopReaderContext() instanceof LeafReaderContext); LeafReaderContext context = (LeafReaderContext) indexSearcher.getTopReaderContext(); - Scorer ts = weight.scorer(context, context.reader().getLiveDocs()); + Scorer ts = weight.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); assertTrue("Didn't skip", ts.advance(3) != DocIdSetIterator.NO_MORE_DOCS); // The next doc should be doc 5 assertTrue("doc should be number 5", ts.docID() == 5); Index: lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (.../branches/lucene2878) (revision 1643331) @@ -19,10 +19,11 @@ import java.io.IOException; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; -import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.PriorityQueue; /** @@ -215,6 +216,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + + @Override public int docID() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } @@ -228,7 +259,7 @@ public int advance(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } - + @Override public long cost() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); @@ -319,7 +350,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanCoord.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanCoord.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanCoord.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,12 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.concurrent.atomic.AtomicBoolean; - import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -36,6 +34,9 @@ import org.junit.AfterClass; import org.junit.BeforeClass; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + /** * Tests coord() computation by BooleanQuery */ @@ -707,7 +708,7 @@ private void assertScore(final float expected, Query query) throws Exception { // test in-order Weight weight = searcher.createNormalizedWeight(query); - Scorer scorer = weight.scorer(reader.leaves().get(0), null); + Scorer scorer = weight.scorer(reader.leaves().get(0), DocsEnum.FLAG_FREQS, null); assertTrue(scorer.docID() == -1 || scorer.docID() == DocIdSetIterator.NO_MORE_DOCS); assertEquals(0, scorer.nextDoc()); assertEquals(expected, scorer.score(), 0.0001f); @@ -715,7 +716,7 @@ // test out-of-order (if supported) if (weight.scoresDocsOutOfOrder()) { final AtomicBoolean seen = new AtomicBoolean(false); - BulkScorer bulkScorer = weight.bulkScorer(reader.leaves().get(0), false, null); + BulkScorer bulkScorer = weight.bulkScorer(reader.leaves().get(0), false, DocsEnum.FLAG_FREQS, null); assertNotNull(bulkScorer); bulkScorer.score(new LeafCollector() { Scorer scorer; Index: lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (.../branches/lucene2878) (revision 1643331) @@ -31,7 +31,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SlowCompositeReaderWrapper; @@ -100,7 +100,7 @@ IndexSearcher searcher = newSearcher(reader); - DocsAndPositionsEnum pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(), + DocsEnum pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(), MultiFields.getLiveDocs(searcher.getIndexReader()), "field", new BytesRef("1")); @@ -212,7 +212,7 @@ final IndexReader readerFromWriter = writer.getReader(); LeafReader r = SlowCompositeReaderWrapper.wrap(readerFromWriter); - DocsAndPositionsEnum tp = r.termPositionsEnum(new Term("content", "a")); + DocsEnum tp = r.termPositionsEnum(new Term("content", "a")); int count = 0; assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java (.../branches/lucene2878) (revision 1643331) @@ -242,6 +242,13 @@ } /* MultiPhraseQuery */ + + /* + "w1 w2 w3 w4 w5", + "w1 w3 w2 w3 zz", + "w1 xx w2 yy w3", + "w1 w3 xx w2 yy w3 zz" + */ public void testMPQ1() throws Exception { MultiPhraseQuery q = new MultiPhraseQuery(); Index: lucene/core/src/java/org/apache/lucene/search/MultiCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MultiCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/MultiCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,11 @@ * limitations under the License. */ +import org.apache.lucene.index.LeafReaderContext; + import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.index.LeafReaderContext; - /** * A {@link Collector} which allows running a search with several * {@link Collector}s. It offers a static {@link #wrap} method which accepts a @@ -101,7 +101,16 @@ return new MultiLeafCollector(leafCollectors); } + @Override + public int postingFeatures() { + int pf = 0; + for (Collector collector : collectors) { + pf |= collector.postingFeatures(); + } + return pf; + } + private static class MultiLeafCollector implements LeafCollector { private final LeafCollector[] collectors; Index: lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/spans/TermSpans.java (.../branches/lucene2878) (revision 1643331) @@ -17,7 +17,7 @@ import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; @@ -30,7 +30,7 @@ * Public for extension only */ public class TermSpans extends Spans { - protected final DocsAndPositionsEnum postings; + protected final DocsEnum postings; protected final Term term; protected int doc; protected int freq; @@ -38,7 +38,7 @@ protected int position; protected boolean readPayload; - public TermSpans(DocsAndPositionsEnum postings, Term term) { + public TermSpans(DocsEnum postings, Term term) { this.postings = postings; this.term = term; doc = -1; @@ -132,7 +132,7 @@ (doc == -1 ? "START" : (doc == Integer.MAX_VALUE) ? "END" : doc + "-" + position); } - public DocsAndPositionsEnum getPostings() { + public DocsEnum getPostings() { return postings; } Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java (.../branches/lucene2878) (revision 1643331) @@ -46,6 +46,7 @@ DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier, Scorer[] subScorers) { super(weight, subScorers); this.tieBreakerMultiplier = tieBreakerMultiplier; + } @Override @@ -66,4 +67,5 @@ protected float getFinal() { return scoreMax + (scoreSum - scoreMax) * tieBreakerMultiplier; } + } Index: lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,16 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ToStringUtils; -import org.apache.lucene.util.Bits; -import java.util.Set; -import java.io.IOException; - /** * A query that matches all documents. * @@ -73,6 +74,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); // nocommit + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int target) throws IOException { doc = target-1; return nextDoc(); @@ -114,7 +145,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new MatchAllScorer(context.reader(), acceptDocs, this, queryWeight); } Index: lucene/core/src/java/org/apache/lucene/search/Weight.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/Weight.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/Weight.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,13 @@ * limitations under the License. */ -import java.io.IOException; - +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReaderContext; // javadocs import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.Bits; +import java.io.IOException; + /** * Expert: Calculate query weights and build query scorers. *

    @@ -34,7 +34,7 @@ * {@link org.apache.lucene.index.LeafReader} dependent state should reside in the {@link Scorer}. *

    * Since {@link Weight} creates {@link Scorer} instances for a given - * {@link org.apache.lucene.index.LeafReaderContext} ({@link #scorer(org.apache.lucene.index.LeafReaderContext, Bits)}) + * {@link org.apache.lucene.index.LeafReaderContext} ({@link #scorer(org.apache.lucene.index.LeafReaderContext, int, Bits)}) * callers must maintain the relationship between the searcher's top-level * {@link IndexReaderContext} and the context used to create a {@link Scorer}. *

    @@ -49,7 +49,7 @@ *

  • The query normalization factor is passed to {@link #normalize(float, float)}. At * this point the weighting is complete. *
  • A Scorer is constructed by - * {@link #scorer(org.apache.lucene.index.LeafReaderContext, Bits)}. + * {@link #scorer(org.apache.lucene.index.LeafReaderContext, int, Bits)}. * * * @since 2.9 @@ -96,7 +96,7 @@ * @return a {@link Scorer} which scores documents in/out-of order. * @throws IOException if there is a low-level I/O error */ - public abstract Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException; + public abstract Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException; /** * Optional method, to return a {@link BulkScorer} to @@ -125,9 +125,9 @@ * passes them to a collector. * @throws IOException if there is a low-level I/O error */ - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { - Scorer scorer = scorer(context, acceptDocs); + Scorer scorer = scorer(context, flags, acceptDocs); if (scorer == null) { // No docs match return null; @@ -198,7 +198,7 @@ * Returns true iff this implementation scores docs only out of order. This * method is used in conjunction with {@link Collector}'s * {@link LeafCollector#acceptsDocsOutOfOrder() acceptsDocsOutOfOrder} and - * {@link #bulkScorer(org.apache.lucene.index.LeafReaderContext, boolean, Bits)} to + * {@link #bulkScorer(org.apache.lucene.index.LeafReaderContext, boolean, int, Bits)} to * create a matching {@link Scorer} instance for a given {@link Collector}, or * vice versa. *

    @@ -205,7 +205,6 @@ * NOTE: the default implementation returns false, i.e. * the Scorer scores documents in-order. */ - public boolean scoresDocsOutOfOrder() { - return false; - } + public boolean scoresDocsOutOfOrder() { return false; } + } Index: lucene/core/src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,18 +17,18 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefIterator; +import java.io.IOException; + /** Iterator to seek ({@link #seekCeil(BytesRef)}, {@link * #seekExact(BytesRef)}) or step through ({@link * #next} terms to obtain frequency information ({@link * #docFreq}), {@link DocsEnum} or {@link - * DocsAndPositionsEnum} for the current term ({@link + * DocsEnum} for the current term ({@link * #docs}. * *

    Term enumerations are always ordered by @@ -162,7 +162,7 @@ * @see #docs(Bits, DocsEnum, int) */ public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException; - /** Get {@link DocsAndPositionsEnum} for the current term. + /** Get {@link DocsEnum} for the current term. * Do not call this when the enum is unpositioned. This * method will return null if positions were not * indexed. @@ -169,13 +169,13 @@ * * @param liveDocs unset bits are documents that should not * be returned - * @param reuse pass a prior DocsAndPositionsEnum for possible reuse - * @see #docsAndPositions(Bits, DocsAndPositionsEnum, int) */ - public final DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { - return docsAndPositions(liveDocs, reuse, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS); + * @param reuse pass a prior DocsEnum for possible reuse + **/ + public final DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse) throws IOException { + return docsAndPositions(liveDocs, reuse, DocsEnum.FLAG_OFFSETS | DocsEnum.FLAG_PAYLOADS); } - /** Get {@link DocsAndPositionsEnum} for the current term, + /** Get {@link DocsEnum} for the current term, * with control over whether offsets and payloads are * required. Some codecs may be able to optimize their * implementation when offsets and/or payloads are not required. @@ -184,11 +184,11 @@ * @param liveDocs unset bits are documents that should not * be returned - * @param reuse pass a prior DocsAndPositionsEnum for possible reuse + * @param reuse pass a prior DocsEnum for possible reuse * @param flags specifies which optional per-position values you - * require; see {@link DocsAndPositionsEnum#FLAG_OFFSETS} and - * {@link DocsAndPositionsEnum#FLAG_PAYLOADS}. */ - public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException; + * require; see {@link DocsEnum#FLAG_OFFSETS} and + * {@link DocsEnum#FLAG_PAYLOADS}. */ + public abstract DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException; /** * Expert: Returns the TermsEnums internal state to position the TermsEnum @@ -250,11 +250,6 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { - throw new IllegalStateException("this method should never be called"); - } - - @Override public BytesRef next() { return null; } @@ -273,5 +268,11 @@ public void seekExact(BytesRef term, TermState state) { throw new IllegalStateException("this method should never be called"); } + + @Override + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) + throws IOException { + throw new IllegalStateException("this method should never be called"); + } }; } Index: lucene/core/src/java/org/apache/lucene/index/DocsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -21,7 +21,8 @@ import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.AttributeSource; -import org.apache.lucene.util.Bits; // javadocs +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; /** Iterates through the documents and term freqs. * NOTE: you must first call {@link #nextDoc} before using @@ -30,9 +31,7 @@ /** * Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} if you don't - * require term frequencies in the returned enum. When passed to - * {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} means - * that no offsets and payloads will be returned. + * require term frequencies in the returned enum. */ public static final int FLAG_NONE = 0x0; @@ -40,6 +39,20 @@ * if you require term frequencies in the returned enum. */ public static final int FLAG_FREQS = 0x1; + /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} + * if you require term positions in the returned enum. */ + public static final int FLAG_POSITIONS = 0x3; + + /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} + * if you require offsets in the returned enum. */ + public static final int FLAG_OFFSETS = 0x7; + + /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)} + * if you require payloads in the returned enum. */ + public static final int FLAG_PAYLOADS = 0xB; + + public static final int NO_MORE_POSITIONS = Integer.MAX_VALUE; + private AttributeSource atts = null; /** Sole constructor. (For invocation by subclass @@ -64,4 +77,31 @@ if (atts == null) atts = new AttributeSource(); return atts; } + + /** Returns the next position. You should only call this + * up to {@link DocsEnum#freq()} times else + * the behavior is not defined. If positions were not + * indexed this will return -1; this only happens if + * offsets were indexed and you passed needsOffset=true + * when pulling the enum. */ + public abstract int nextPosition() throws IOException; + + public abstract int startPosition() throws IOException; + + public abstract int endPosition() throws IOException; + + /** Returns start offset for the current position, or -1 + * if offsets were not indexed. */ + public abstract int startOffset() throws IOException; + + /** Returns end offset for the current position, or -1 if + * offsets were not indexed. */ + public abstract int endOffset() throws IOException; + + /** Returns the payload at this position, or null if no + * payload was indexed. You should not modify anything + * (neither members of the returned BytesRef nor bytes + * in the byte[]). */ + public abstract BytesRef getPayload() throws IOException; + } Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (.../branches/lucene2878) (revision 1643331) @@ -884,7 +884,7 @@ DocsEnum docs = null; DocsEnum docsAndFreqs = null; - DocsAndPositionsEnum postings = null; + DocsEnum postings = null; String lastField = null; for (String field : fields) { @@ -1807,11 +1807,11 @@ } DocsEnum docs = null; - DocsAndPositionsEnum postings = null; + DocsEnum postings = null; // Only used if crossCheckTermVectors is true: DocsEnum postingsDocs = null; - DocsAndPositionsEnum postingsPostings = null; + DocsEnum postingsPostings = null; final Bits liveDocs = reader.getLiveDocs(); Index: lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,6 +17,8 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -26,8 +28,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import java.io.IOException; - /** This class only tests some basic functionality in CSQ, the main parts are mostly * tested by MultiTermQuery tests, explanations seems to be tested in TestExplanations! */ public class TestConstantScoreQuery extends LuceneTestCase { @@ -56,9 +56,9 @@ public void setScorer(Scorer scorer) { this.scorer = scorer; assertEquals("Scorer is implemented by wrong class", scorerClassName, scorer.getClass().getName()); - if (innerScorerClassName != null && scorer instanceof ConstantScoreQuery.ConstantScorer) { - final ConstantScoreQuery.ConstantScorer innerScorer = (ConstantScoreQuery.ConstantScorer) scorer; - assertEquals("inner Scorer is implemented by wrong class", innerScorerClassName, innerScorer.docIdSetIterator.getClass().getName()); + if (innerScorerClassName != null && scorer instanceof ConstantScoreQuery.ConstantScoreScorer) { + final ConstantScoreQuery.ConstantScoreScorer innerScorer = (ConstantScoreQuery.ConstantScoreScorer) scorer; + assertEquals("inner Scorer is implemented by wrong class", innerScorerClassName, innerScorer.in.getClass().getName()); } } @@ -113,13 +113,13 @@ final Query csqbq = new ConstantScoreQuery(bq); csqbq.setBoost(17.0f); - checkHits(searcher, csq1, csq1.getBoost(), ConstantScoreQuery.ConstantScorer.class.getName(), null); - checkHits(searcher, csq2, csq2.getBoost(), ConstantScoreQuery.ConstantScorer.class.getName(), ConstantScoreQuery.ConstantScorer.class.getName()); + checkHits(searcher, csq1, csq1.getBoost(), ConstantScoreQuery.ConstantScoreScorer.class.getName(), null); + checkHits(searcher, csq2, csq2.getBoost(), ConstantScoreQuery.ConstantScoreScorer.class.getName(), ConstantScoreQuery.ConstantScoreScorer.class.getName()); // for the combined BQ, the scorer should always be BooleanScorer's BucketScorer, because our scorer supports out-of order collection! final String bucketScorerClass = FakeScorer.class.getName(); checkHits(searcher, bq, csq1.getBoost() + csq2.getBoost(), bucketScorerClass, null); - checkHits(searcher, csqbq, csqbq.getBoost(), ConstantScoreQuery.ConstantScorer.class.getName(), bucketScorerClass); + checkHits(searcher, csqbq, csqbq.getBoost(), ConstantScoreQuery.ConstantScoreScorer.class.getName(), bucketScorerClass); } finally { if (reader != null) reader.close(); if (directory != null) directory.close(); Index: lucene/core/src/test/org/apache/lucene/search/posfilter/IntervalTestBase.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/IntervalTestBase.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/IntervalTestBase.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,259 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CheckHits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PositionsCollector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; + +import java.io.IOException; + +public abstract class IntervalTestBase extends LuceneTestCase { + + protected Directory directory; + protected IndexReader reader; + protected IndexSearcher searcher; + + public static class AssertingPositionsCollector extends PositionsCollector { + + enum AssertionType { POSITIONS, OFFSETS } + + private final int[][] expectedResults; + private final AssertionType type; + + private int docUpto = -1; + private int posUpto = -1; + + private int currentDoc = -1; + private int posRemaining = 0; + + public AssertingPositionsCollector(int[][] expectedResults, AssertionType type) { + this.expectedResults = expectedResults; + this.type = type; + } + + @Override + public int postingFeatures() { + if (type == AssertionType.POSITIONS) + return DocsEnum.FLAG_POSITIONS; + else + return DocsEnum.FLAG_OFFSETS; + } + + @Override + protected void collectPosition(int doc, Interval interval) { + + if (doc != currentDoc) { + if (posRemaining > 0) { + int missingPos = expectedResults[docUpto].length - (posRemaining * 2); + fail("Missing expected hit in document " + expectedResults[docUpto][0] + ": [" + + expectedResults[docUpto][missingPos] + ", " + expectedResults[docUpto][missingPos + 1] + "]"); + } + docUpto++; + if (docUpto > expectedResults.length - 1) + fail("Unexpected hit in document " + doc + ": " + interval.toString()); + + currentDoc = expectedResults[docUpto][0]; + posUpto = -1; + posRemaining = (expectedResults[docUpto].length - 1) / 2; + } + + if (doc != currentDoc) + fail("Expected next hit in document " + currentDoc + " but was in " + doc + ": " + interval.toString()); + + posUpto++; + posRemaining--; + + if (posUpto > ((expectedResults[docUpto].length - 1) / 2) - 1) + fail("Unexpected hit in document " + doc + ": " + interval.toString()); + + if (type == AssertionType.POSITIONS) { + if (expectedResults[docUpto][posUpto * 2 + 1] != interval.begin || + expectedResults[docUpto][posUpto * 2 + 2] != interval.end) + fail("Expected next position in document to be [" + expectedResults[docUpto][posUpto * 2 + 1] + ", " + + expectedResults[docUpto][posUpto * 2 + 2] + "] but was [" + interval.begin + ", " + interval.end + "]"); + } + else { + // check offsets + if (expectedResults[docUpto][posUpto * 2 + 1] != interval.offsetBegin || + expectedResults[docUpto][posUpto * 2 + 2] != interval.offsetEnd) + fail("Expected next offset in document to be [" + expectedResults[docUpto][posUpto * 2 + 1] + ", " + + expectedResults[docUpto][posUpto * 2 + 2] + "] but was [" + interval.offsetBegin + ", " + interval.offsetEnd + "]"); + } + } + + public void assertAllMatched() { + if (docUpto < expectedResults.length - 1) { + fail("Expected a hit in document " + expectedResults[docUpto + 1][0]); + } + } + } + + /** + * Run a query against a searcher, and check that the collected intervals from the query match + * the expected results. + * @param q the query + * @param searcher the searcher + * @param expectedResults an int[][] detailing the expected results, in the format + * { { docid1, startoffset1, endoffset1, startoffset2, endoffset2, ... }, + * { docid2, startoffset1, endoffset1, startoffset2, endoffset2, ...}, ... } + * @throws IOException + */ + public static void checkIntervalOffsets(Query q, IndexSearcher searcher, int[][] expectedResults) throws IOException { + + //MatchCollector m = new MatchCollector(); + AssertingPositionsCollector c = new AssertingPositionsCollector(expectedResults, AssertingPositionsCollector.AssertionType.OFFSETS); + searcher.search(q, c); + c.assertAllMatched(); + + } + + /** + * Run a query against a searcher, and check that the collected intervals from the query match + * the expected results. + * @param q the query + * @param searcher the searcher + * @param expectedResults an int[][] detailing the expected results, in the format + * { { docid1, startpos1, endpos1, startpos2, endpos2, ... }, + * { docid2, startpos1, endpos1, startpos2, endpos2, ...}, ... } + * @throws IOException + */ + public static void checkIntervals(Query q, IndexSearcher searcher, int[][] expectedResults) throws IOException { + + AssertingPositionsCollector c = new AssertingPositionsCollector(expectedResults, AssertingPositionsCollector.AssertionType.POSITIONS); + searcher.search(q, c); + c.assertAllMatched(); + + } + + public static void checkScores(Query q, IndexSearcher searcher, int... expectedDocs) throws IOException { + TopDocs hits = searcher.search(q, 1000); + Assert.assertEquals("Wrong number of hits", expectedDocs.length, hits.totalHits); + for (int i = 0; i < expectedDocs.length; i++) { + Assert.assertEquals("Docs not scored in order", expectedDocs[i], hits.scoreDocs[i].doc); + } + CheckHits.checkExplanations(q, "field", searcher); + } + + protected abstract void addDocs(RandomIndexWriter writer) throws IOException; + + @Before + public void setUp() throws Exception { + super.setUp(); + directory = newDirectory(); + IndexWriterConfig config = newIndexWriterConfig(new MockAnalyzer(random())); + //config.setCodec(Codec.forName("SimpleText")); + //config.setCodec(Codec.forName("Asserting")); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory, config); + addDocs(writer); + reader = writer.getReader(); + writer.close(); + searcher = new IndexSearcher(reader); + } + + @After + public void tearDown() throws Exception { + reader.close(); + directory.close(); + super.tearDown(); + } + + public TermQuery makeTermQuery(String text) { + return new TermQuery(new Term(TestBasicIntervals.field, text)); + } + + protected Query makeOrQuery(Query... queries) { + BooleanQuery q = new BooleanQuery(); + for (Query subquery : queries) { + q.add(subquery, BooleanClause.Occur.SHOULD); + } + return q; + } + + protected Query makeAndQuery(Query... queries) { + BooleanQuery q = new BooleanQuery(); + for (Query subquery : queries) { + q.add(subquery, BooleanClause.Occur.MUST); + } + return q; + } + + protected Query makeBooleanQuery(BooleanClause... clauses) { + BooleanQuery q = new BooleanQuery(); + for (BooleanClause clause : clauses) { + q.add(clause); + } + return q; + } + + protected BooleanClause makeBooleanClause(String text, BooleanClause.Occur occur) { + return new BooleanClause(makeTermQuery(text), occur); + } + + public static class Match implements Comparable { + + public final int docid; + public final int start; + public final int end; + public final int startOffset; + public final int endOffset; + public final boolean composite; + + public Match(int docid, Interval interval, boolean composite) { + this.docid = docid; + this.start = interval.begin; + this.end = interval.end; + this.startOffset = interval.offsetBegin; + this.endOffset = interval.offsetEnd; + this.composite = composite; + } + + @Override + public int compareTo(Match o) { + if (this.docid != o.docid) + return this.docid - o.docid; + if (this.start != o.start) + return this.start - o.start; + return o.end - this.end; + } + + @Override + public String toString() { + return String.format("%d:%d[%d]->%d[%d]%s", + docid, start, startOffset, end, endOffset, composite ? "C" : ""); + } + } + + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/IntervalTestBase.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestRangeFilterQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestRangeFilterQuery.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestRangeFilterQuery.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,72 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.posfilter.OrderedNearQuery; +import org.apache.lucene.search.posfilter.RangeFilterQuery; +import org.junit.Test; + +import java.io.IOException; + +public class TestRangeFilterQuery extends IntervalTestBase { + + @Override + protected void addDocs(RandomIndexWriter writer) throws IOException { + for (int i = 0; i < docFields.length; i++) { + Document doc = new Document(); + doc.add(newField("field", docFields[i], TextField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + private String[] docFields = { + "w1 w2 w3 w4 w5 w6 w7 w8 w9 w10 w11 w12", //0 + "w1 w3 w4 w5 w6 w7 w8 w4", //1 + "w1 w3 w10 w4 w5 w6 w7 w8", //2 + "w1 w3 w2 w4 w10 w5 w6 w7 w8", //3 + }; + + @Test + public void testSimpleTermRangeFilter() throws IOException { + Query q = new RangeFilterQuery(2, makeTermQuery("w4")); + checkIntervals(q, searcher, new int[][]{ + { 1, 2, 2 } + }); + } + + @Test + public void testStartEndTermRangeFilter() throws IOException { + Query q = new RangeFilterQuery(2, 4, makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 2, 2 } + }); + } + + public void testRangeFilteredPositionFilter() throws IOException { + Query q = new OrderedNearQuery(0, makeTermQuery("w4"), makeTermQuery("w5")); + q = new RangeFilterQuery(3, 10, q); + checkIntervals(q, searcher, new int[][]{ + { 0, 3, 4 }, + { 2, 3, 4 } + }); + } +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestRangeFilterQuery.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionFilteredIntervals.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionFilteredIntervals.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionFilteredIntervals.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,224 @@ +package org.apache.lucene.search.posfilter; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; + +public class TestPositionFilteredIntervals extends IntervalTestBase { + + public static final String field = "field"; + + @Override + protected void addDocs(RandomIndexWriter writer) throws IOException { + for (String content : docFields) { + Document doc = new Document(); + doc.add(newField(field, content, TextField.TYPE_NOT_STORED)); + writer.addDocument(doc); + } + } + + private String[] docFields = { + "w1 w2 w3 w4 w5", //0 + "w1 w3 w2 w3",//1 + "w1 xx w2 yy w3",//2 + "w1 w3 xx w2 yy w3",//3 + "u2 u2 u1", //4 + "u2 xx u2 u1",//5 + "u2 u2 xx u1", //6 + "u2 xx u2 yy u1", //7 + "u2 xx u1 u2",//8 + "u1 u2 xx u2",//9 + "u2 u1 xx u2",//10 + "t1 t2 t1 t3 t2 t3",//11 + "v1 v2 v3",//12 + "v1 v3 v2 v3 v4",//13 + "v4 v2 v2 v4",//14 + "v3 v4 v3"};//15 + + public void testNearOrdered01() throws Exception { + Query q = new OrderedNearQuery(0, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 } + }); + } + + public void testNearOrdered02() throws Exception { + Query q = new OrderedNearQuery(1, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 }, + { 1, 0, 3 } + }); + } + + public void testNearOrdered03() throws Exception { + Query q = new OrderedNearQuery(2, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 }, + { 1, 0, 3 }, + { 2, 0, 4 } + }); + } + + public void testNearOrdered04() throws Exception { + Query q = new OrderedNearQuery(3, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 }, + { 1, 0, 3 }, + { 2, 0, 4 }, + { 3, 0, 5 } + }); + } + + public void testNearOrdered05() throws Exception { + Query q = new OrderedNearQuery(4, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 }, + { 1, 0, 3 }, + { 2, 0, 4 }, + { 3, 0, 5 } + }); + } + + public void testNearOrderedEqual01() throws Exception { + Query q = new OrderedNearQuery(0, makeTermQuery("w1"), makeTermQuery("w3"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{}); + } + + public void testNearOrderedEqual02() throws Exception { + Query q = new OrderedNearQuery(1, makeTermQuery("w1"), makeTermQuery("w3"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 1, 0, 3 } + }); + } + + public void testNearOrderedEqual03() throws Exception { + Query q = new OrderedNearQuery(2, makeTermQuery("w1"), makeTermQuery("w3"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 1, 0, 3 } + }); + } + + public void testNearOrderedEqual04() throws Exception { + Query q = new OrderedNearQuery(3, makeTermQuery("w1"), makeTermQuery("w3"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 1, 0, 3 }, + { 3, 0, 5 } + }); + } + + public void testNearOrderedEqual11() throws Exception { + Query q = new OrderedNearQuery(0, makeTermQuery("u2"), makeTermQuery("u2"), makeTermQuery("u1")); + checkIntervals(q, searcher, new int[][]{ + { 4, 0, 2 } + }); + } + + public void testNearOrderedEqual13() throws Exception { + Query q = new OrderedNearQuery(1, makeTermQuery("u2"), makeTermQuery("u2"), makeTermQuery("u1")); + checkIntervals(q, searcher, new int[][]{ + { 4, 0, 2 }, + { 5, 0, 3 }, + { 6, 0, 3 } + }); + } + + public void testNearOrderedEqual14() throws Exception { + Query q = new OrderedNearQuery(2, makeTermQuery("u2"), makeTermQuery("u2"), makeTermQuery("u1")); + checkIntervals(q, searcher, new int[][]{ + { 4, 0, 2 }, + { 5, 0, 3 }, + { 6, 0, 3 }, + { 7, 0, 4 } + }); + } + + public void testNearOrderedEqual15() throws Exception { + Query q = new OrderedNearQuery(3, makeTermQuery("u2"), makeTermQuery("u2"), makeTermQuery("u1")); + checkIntervals(q, searcher, new int[][]{ + { 4, 0, 2 }, + { 5, 0, 3 }, + { 6, 0, 3 }, + { 7, 0, 4 } + }); + } + + public void testNearOrderedOverlap() throws Exception { + Query q = new OrderedNearQuery(3, makeTermQuery("t1"), makeTermQuery("t2"), makeTermQuery("t3")); + checkIntervals(q, searcher, new int[][]{ + { 11, 0, 3, 2, 5 } + }); + } + + public void testNearUnordered() throws Exception { + Query q = new UnorderedNearQuery(0, makeTermQuery("u1"), makeTermQuery("u2")); + checkIntervals(q, searcher, new int[][]{ + { 4, 1, 2 }, + { 5, 2, 3 }, + { 8, 2, 3 }, + { 9, 0, 1 }, + { 10, 0, 1 } + }); + } + + public void testMultipleNearUnordered() throws Exception { + Query q = new UnorderedNearQuery(1, makeTermQuery("w1"), makeTermQuery("w2"), makeTermQuery("w3")); + checkIntervals(q, searcher, new int[][]{ + { 0, 0, 2 }, + { 1, 0, 2 }, + { 3, 0, 3 } + }); + } + /* + "w1 w2 w3 w4 w5", //0 + "w1 w3 w2 w3",//1 + "w1 xx w2 yy w3",//2 + "w1 w3 xx w2 yy w3",//3 + "u2 u2 u1", //4 + "u2 xx u2 u1",//5 + "u2 u2 xx u1", //6 + "u2 xx u2 yy u1", //7 + "u2 xx u1 u2",//8 + "u1 u2 xx u2",//9 + "u2 u1 xx u2",//10 + "t1 t2 t1 t3 t2 t3"};//11 + */ + + // ((u1 near u2) and xx) + public void testNestedNear() throws Exception { + + Query q = new UnorderedNearQuery(0, makeTermQuery("u1"), makeTermQuery("u2")); + BooleanQuery topq = new BooleanQuery(); + topq.add(q, BooleanClause.Occur.MUST); + topq.add(makeTermQuery("xx"), BooleanClause.Occur.MUST); + + checkIntervals(topq, searcher, new int[][]{ + { 5, 1, 1, 2, 3 }, + { 8, 1, 1, 2, 3 }, + { 9, 0, 1, 2, 2 }, + { 10, 0, 1, 2, 2 } + }); + + } +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionFilteredIntervals.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/spans/TestNearSpansOrdered.java (.../branches/lucene2878) (revision 1643331) @@ -20,16 +20,17 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.Scorer; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -183,7 +184,7 @@ Weight w = searcher.createNormalizedWeight(q); IndexReaderContext topReaderContext = searcher.getTopReaderContext(); LeafReaderContext leave = topReaderContext.leaves().get(0); - Scorer s = w.scorer(leave, leave.reader().getLiveDocs()); + Scorer s = w.scorer(leave, DocsEnum.FLAG_POSITIONS, leave.reader().getLiveDocs()); assertEquals(1, s.advance(1)); } Index: lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (.../branches/lucene2878) (revision 1643331) @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; public class TestCachingCollector extends LuceneTestCase { @@ -38,6 +40,36 @@ public int freq() throws IOException { return 0; } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public int docID() { return 0; } @Override @@ -45,7 +77,7 @@ @Override public int advance(int target) throws IOException { return 0; } - + @Override public long cost() { return 1; @@ -170,7 +202,7 @@ public void testNoWrappedCollector() throws Exception { for (boolean cacheScores : new boolean[] { false, true }) { // create w/ null wrapped collector, and test that the methods work - CachingCollector cc = CachingCollector.create(true, cacheScores, 50 * ONE_BYTE); + CachingCollector cc = CachingCollector.create(true, DocsEnum.FLAG_NONE, cacheScores, 50 * ONE_BYTE); LeafCollector acc = cc.getLeafCollector(null); acc.setScorer(new MockScorer()); acc.collect(0); Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanQueryVisitSubscorers.java (.../branches/lucene2878) (revision 1643331) @@ -17,19 +17,12 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; @@ -40,6 +33,14 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + // TODO: refactor to a base class, that collects freqs from the scorer tree // and test all queries with it public class TestBooleanQueryVisitSubscorers extends LuceneTestCase { @@ -254,6 +255,11 @@ }; } + @Override + public int postingFeatures() { + return DocsEnum.FLAG_NONE; + } + private static void summarizeScorer(final StringBuilder builder, final Scorer scorer, final int indent) { builder.append(scorer.getClass().getSimpleName()); if (scorer instanceof TermScorer) { Index: lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (.../branches/lucene2878) (revision 1643331) @@ -23,6 +23,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; public class TestScoreCachingWrappingScorer extends LuceneTestCase { @@ -47,6 +48,36 @@ return 1; } + @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + @Override public int docID() { return doc; } @Override public int nextDoc() { @@ -57,7 +88,7 @@ doc = target; return doc < scores.length ? doc : NO_MORE_DOCS; } - + @Override public long cost() { return scores.length; Index: lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,21 +17,21 @@ * limitations under the License. */ -import org.apache.lucene.document.Field; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SlowCompositeReaderWrapper; -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.StoredDocument; import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.DefaultSimilarity; @@ -39,11 +39,12 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.Locale; -import java.io.IOException; /** * Test of the DisjunctionMaxQuery. @@ -180,7 +181,7 @@ assertTrue(s.getTopReaderContext() instanceof LeafReaderContext); final Weight dw = s.createNormalizedWeight(dq); LeafReaderContext context = (LeafReaderContext)s.getTopReaderContext(); - final Scorer ds = dw.scorer(context, context.reader().getLiveDocs()); + final Scorer ds = dw.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); final boolean skipOk = ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS; if (skipOk) { fail("firsttime skipTo found a match? ... " @@ -196,7 +197,7 @@ QueryUtils.check(random(), dq, s); final Weight dw = s.createNormalizedWeight(dq); LeafReaderContext context = (LeafReaderContext)s.getTopReaderContext(); - final Scorer ds = dw.scorer(context, context.reader().getLiveDocs()); + final Scorer ds = dw.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); assertTrue("firsttime skipTo found no match", ds.advance(3) != DocIdSetIterator.NO_MORE_DOCS); assertEquals("found wrong docid", "d4", r.document(ds.docID()).get("id")); Index: lucene/core/src/java/org/apache/lucene/search/PositionsCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PositionsCollector.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/PositionsCollector.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,62 @@ +package org.apache.lucene.search; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.posfilter.Interval; + +import java.io.IOException; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public abstract class PositionsCollector extends SimpleCollector { + + private Scorer scorer; + private final boolean collectOffsets; + + protected PositionsCollector(boolean collectOffsets) { + this.collectOffsets = collectOffsets; + } + + protected PositionsCollector() { + this(false); + } + + @Override + public final void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + @Override + public final void collect(int doc) throws IOException { + while (scorer.nextPosition() != DocsEnum.NO_MORE_POSITIONS) { + collectPosition(doc, new Interval(scorer)); + } + } + + protected abstract void collectPosition(int doc, Interval interval); + + @Override + public int postingFeatures() { + return collectOffsets ? DocsEnum.FLAG_OFFSETS : DocsEnum.FLAG_POSITIONS; + } + + @Override + public final boolean acceptsDocsOutOfOrder() { + return false; + } + +} Property changes on: lucene/core/src/java/org/apache/lucene/search/PositionsCollector.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanWeight.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,18 @@ * limitations under the License. */ +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; -import org.apache.lucene.search.*; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TermStatistics; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; import org.apache.lucene.util.Bits; @@ -81,7 +88,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { if (stats == null) { return null; } else { @@ -91,7 +98,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - SpanScorer scorer = (SpanScorer) scorer(context, context.reader().getLiveDocs()); + SpanScorer scorer = (SpanScorer) scorer(context, DocsEnum.FLAG_POSITIONS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (.../branches/lucene2878) (revision 1643331) @@ -20,7 +20,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Fields; import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; @@ -115,7 +115,7 @@ final TermsEnum termsEnum = context.reader().terms(term.field()).iterator(null); termsEnum.seekExact(term.bytes(), state); - final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null, DocsAndPositionsEnum.FLAG_PAYLOADS); + final DocsEnum postings = termsEnum.docsAndPositions(acceptDocs, null, DocsEnum.FLAG_PAYLOADS); if (postings != null) { return new TermSpans(postings, term); Index: lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (.../branches/lucene2878) (revision 1643331) @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.Collection; +import org.apache.lucene.util.BytesRef; + /** A Scorer for queries with a required part and an optional part. * Delays skipTo() on the optional part until a score() is needed. *
    @@ -29,6 +31,7 @@ /** The scorers passed from the constructor. * These are set to null as soon as their next() or skipTo() returns false. */ + private PositionQueue posQueue; protected Scorer reqScorer; protected Scorer optScorer; @@ -45,16 +48,21 @@ assert optScorer != null; this.reqScorer = reqScorer; this.optScorer = optScorer; + posQueue = new PositionQueue(reqScorer, optScorer); } @Override public int nextDoc() throws IOException { - return reqScorer.nextDoc(); + int doc = reqScorer.nextDoc(); + posQueue.advanceTo(doc); + return doc; } @Override public int advance(int target) throws IOException { - return reqScorer.advance(target); + int doc = reqScorer.advance(target); + posQueue.advanceTo(doc); + return doc; } @Override @@ -93,6 +101,39 @@ } @Override + public int nextPosition() throws IOException { + int optDoc = optScorer.docID(); + if (optDoc < reqScorer.docID()) + optScorer.advance(reqScorer.docID()); + return posQueue.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + return posQueue.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return posQueue.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return posQueue.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return posQueue.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return posQueue.getPayload(); + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList<>(2); children.add(new ChildScorer(reqScorer, "MUST")); Index: lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (.../branches/lucene2878) (revision 1643331) @@ -1,6 +1,6 @@ package org.apache.lucene.search; -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -18,24 +18,34 @@ */ import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermContext; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.PhraseQuery.TermDocsEnumFactory; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.search.similarities.Similarity.SimScorer; -import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IntroSorter; import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.ToStringUtils; @@ -179,10 +189,13 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { assert !termArrays.isEmpty(); final LeafReader reader = context.reader(); final Bits liveDocs = acceptDocs; + + if ((flags & DocsEnum.FLAG_POSITIONS) < DocsEnum.FLAG_POSITIONS) + flags = DocsEnum.FLAG_POSITIONS; PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[termArrays.size()]; @@ -197,9 +210,9 @@ for (int pos=0; pos 1) { postingsEnum = new UnionDocsAndPositionsEnum(liveDocs, context, terms, termContexts, termsEnum); @@ -221,6 +234,7 @@ // None of the terms are in this reader return null; } + factory = new MultiTermDocsEnumFactory(liveDocs, context, terms, termContexts, termsEnum, flags); } else { final Term term = terms[0]; TermState termState = termContexts.get(term).get(context.ord); @@ -229,7 +243,7 @@ return null; } termsEnum.seekExact(term.bytes(), termState); - postingsEnum = termsEnum.docsAndPositions(liveDocs, null, DocsEnum.FLAG_NONE); + postingsEnum = termsEnum.docsAndPositions(liveDocs, null, flags); if (postingsEnum == null) { // term does exist, but has no positions @@ -237,10 +251,10 @@ throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); } - docFreq = termsEnum.docFreq(); + factory = new TermDocsEnumFactory(term.bytes(), termState, termsEnum, flags, acceptDocs); } - - postingsFreqs[pos] = new PhraseQuery.PostingsAndFreq(postingsEnum, docFreq, positions.get(pos).intValue(), terms); + + postingsFreqs[pos] = new PhraseQuery.PostingsAndFreq(postingsEnum, factory, termsEnum.docFreq() , positions.get(pos).intValue(), terms); } // sort by increasing docFreq order @@ -257,7 +271,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context, context.reader().getLiveDocs()); + Scorer scorer = scorer(context, DocsEnum.FLAG_POSITIONS, context.reader().getLiveDocs()); if (scorer != null) { int newDoc = scorer.advance(doc); if (newDoc == doc) { @@ -401,6 +415,27 @@ } return true; } + + private static class MultiTermDocsEnumFactory extends TermDocsEnumFactory { + + LeafReaderContext context; + Term[] terms; + Map termContexts; + + MultiTermDocsEnumFactory(Bits liveDocs, LeafReaderContext context, Term[] terms, + Map termContexts, TermsEnum termsEnum, int flags) throws IOException { + super(termsEnum, flags, liveDocs); + this.context = context; + this.terms = terms; + this.termContexts = termContexts; + } + + @Override + public DocsEnum docsAndPositionsEnum() throws IOException { + return new UnionDocsAndPositionsEnum(liveDocs, context, terms, termContexts, termsEnum, flags); + } + + } } /** @@ -408,15 +443,15 @@ */ // TODO: if ever we allow subclassing of the *PhraseScorer -class UnionDocsAndPositionsEnum extends DocsAndPositionsEnum { +class UnionDocsAndPositionsEnum extends DocsEnum { - private static final class DocsQueue extends PriorityQueue { - DocsQueue(List docsEnums) throws IOException { + private static final class DocsQueue extends PriorityQueue { + DocsQueue(List docsEnums) throws IOException { super(docsEnums.size()); - Iterator i = docsEnums.iterator(); + Iterator i = docsEnums.iterator(); while (i.hasNext()) { - DocsAndPositionsEnum postings = i.next(); + DocsEnum postings = i.next(); if (postings.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { add(postings); } @@ -424,30 +459,54 @@ } @Override - public final boolean lessThan(DocsAndPositionsEnum a, DocsAndPositionsEnum b) { + public final boolean lessThan(DocsEnum a, DocsEnum b) { return a.docID() < b.docID(); } } - private static final class IntQueue { - private int _arraySize = 16; + // TODO: Reimplement this as int[_arraySize * 3], storing position at i * 3, + // startOffset at i * 3 + 1 and endOffset at i * 3 + 2. Will need to also + // implement a new SorterTemplate to sort the array. + + private static final class PositionQueue { + private int _arraySize = 48; private int _index = 0; private int _lastIndex = 0; private int[] _array = new int[_arraySize]; - final void add(int i) { - if (_lastIndex == _arraySize) + final void add(int pos, int start, int end) { + if (_lastIndex * 3 == _arraySize) growArray(); - _array[_lastIndex++] = i; + _array[_lastIndex * 3] = pos; + _array[_lastIndex * 3 + 1] = start; + _array[_lastIndex * 3 + 2] = end; + _lastIndex += 1; } final int next() { - return _array[_index++]; + return _array[_index++ * 3]; } + final int startPosition() { + return _array[(_index - 1) * 3]; + } + + final int endPosition() { + return _array[(_index - 1) * 3]; + } + + final int startOffset() { + return _array[(_index - 1) * 3 + 1]; + } + + final int endOffset() { + return _array[(_index - 1) * 3 + 2]; + } + final void sort() { - Arrays.sort(_array, _index, _lastIndex); + //Arrays.sort(_array, _index, _lastIndex); + sorter.sort(_index, _lastIndex); } final void clear() { @@ -465,16 +524,54 @@ _array = newArray; _arraySize *= 2; } + + private IntroSorter sorter = new IntroSorter() { + private int pivot; + + @Override + protected void swap(int i, int j) { + int ti = _array[i * 3]; + int ts = _array[i * 3 + 1]; + int te = _array[i * 3 + 2]; + _array[i * 3] = _array[j * 3]; + _array[i * 3 + 1] = _array[j * 3 + 1]; + _array[i * 3 + 2] = _array[j * 3 + 2]; + _array[j * 3] = ti; + _array[j * 3 + 1] = ts; + _array[j * 3 + 2] = te; + } + + @Override + protected int compare(int i, int j) { + return _array[i * 3] - _array[j * 3]; + } + + @Override + protected void setPivot(int i) { + pivot = i; + } + + @Override + protected int comparePivot(int j) { + return pivot - _array[j * 3]; + } + }; } private int _doc = -1; private int _freq; private DocsQueue _queue; - private IntQueue _posList; + private PositionQueue _posList; + private int posPending; private long cost; - public UnionDocsAndPositionsEnum(Bits liveDocs, LeafReaderContext context, Term[] terms, Map termContexts, TermsEnum termsEnum) throws IOException { - List docsEnums = new LinkedList<>(); + public UnionDocsAndPositionsEnum(Bits liveDocs, LeafReaderContext context, Term[] terms, + Map termContexts, TermsEnum termsEnum) throws IOException { + this(liveDocs, context, terms, termContexts, termsEnum, DocsEnum.FLAG_POSITIONS); + } + + public UnionDocsAndPositionsEnum(Bits liveDocs, LeafReaderContext context, Term[] terms, Map termContexts, TermsEnum termsEnum, int flags) throws IOException { + List docsEnums = new LinkedList<>(); for (int i = 0; i < terms.length; i++) { final Term term = terms[i]; TermState termState = termContexts.get(term).get(context.ord); @@ -483,7 +580,7 @@ continue; } termsEnum.seekExact(term.bytes(), termState); - DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, DocsEnum.FLAG_NONE); + DocsEnum postings = termsEnum.docsAndPositions(liveDocs, null, flags); if (postings == null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); @@ -493,7 +590,7 @@ } _queue = new DocsQueue(docsEnums); - _posList = new IntQueue(); + _posList = new PositionQueue(); } @Override @@ -509,13 +606,13 @@ _doc = _queue.top().docID(); // merge sort all positions together - DocsAndPositionsEnum postings; + DocsEnum postings; do { postings = _queue.top(); final int freq = postings.freq(); for (int i = 0; i < freq; i++) { - _posList.add(postings.nextPosition()); + _posList.add(postings.nextPosition(), postings.startOffset(), postings.endOffset()); } if (postings.nextDoc() != NO_MORE_DOCS) { @@ -527,6 +624,7 @@ _posList.sort(); _freq = _posList.size(); + posPending = _freq; return _doc; } @@ -533,17 +631,30 @@ @Override public int nextPosition() { + if (posPending == 0) + return NO_MORE_POSITIONS; + posPending--; return _posList.next(); } @Override + public int startPosition() throws IOException { + return _posList.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return _posList.endPosition(); + } + + @Override public int startOffset() { - return -1; + return _posList.startOffset(); } @Override public int endOffset() { - return -1; + return _posList.endOffset(); } @Override @@ -554,7 +665,7 @@ @Override public final int advance(int target) throws IOException { while (_queue.top() != null && target > _queue.top().docID()) { - DocsAndPositionsEnum postings = _queue.pop(); + DocsEnum postings = _queue.pop(); if (postings.advance(target) != NO_MORE_DOCS) { _queue.add(postings); } @@ -563,7 +674,7 @@ } @Override - public final int freq() { + public final int freq() throws IOException { return _freq; } Index: lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -22,8 +22,8 @@ import java.util.Comparator; import java.util.List; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FilteredTermsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; @@ -271,8 +271,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, - DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, + DocsEnum reuse, int flags) throws IOException { return actualEnum.docsAndPositions(liveDocs, reuse, flags); } Index: lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java (.../trunk) (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,109 @@ +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.search.posfilter.Interval; +import org.apache.lucene.util.BytesRef; + +/** + * Copyright (c) 2014 Lemur Consulting Ltd. + *

    + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

    + * http://www.apache.org/licenses/LICENSE-2.0 + *

    + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public abstract class PhraseScorer extends Scorer { + + protected PhraseScorer(Weight weight) { + super(weight); + } + + protected int freq = -1; + protected Interval[] positionCache = new Interval[4]; + private int currentPos = -1; + + @Override + public final int freq() throws IOException { + if (freq == -1) { + cachePositions(); + } + return freq; + } + + private void cachePositions() throws IOException { + assert freq == -1; + int f = 0; + while (doNextPosition() != NO_MORE_POSITIONS) { + if (f >= positionCache.length) { + Interval[] newCache = new Interval[positionCache.length * 2]; + System.arraycopy(positionCache, 0, newCache, 0, positionCache.length); + positionCache = newCache; + } + positionCache[f] = new Interval(this); + f++; + } + this.freq = f; + } + + @Override + public final int nextPosition() throws IOException { + if (freq == -1) + return doNextPosition(); + currentPos++; + if (currentPos >= freq) + return NO_MORE_POSITIONS; + return positionCache[currentPos].begin; + } + + @Override + public final int startPosition() throws IOException { + if (freq == -1) + return doStartPosition(); + return positionCache[currentPos].begin; + } + + @Override + public final int endPosition() throws IOException { + if (freq == -1) + return doEndPosition(); + return positionCache[currentPos].end; + } + + @Override + public final int startOffset() throws IOException { + if (freq == -1) + return doStartOffset(); + return positionCache[currentPos].offsetBegin; + } + + @Override + public final int endOffset() throws IOException { + if (freq == -1) + return doEndOffset(); + return positionCache[currentPos].offsetEnd; + } + + @Override + public final BytesRef getPayload() throws IOException { + return null; // TODO - how to deal with payloads on intervals? + } + + protected abstract int doNextPosition() throws IOException; + + protected abstract int doStartPosition() throws IOException; + + protected abstract int doEndPosition() throws IOException; + + protected abstract int doStartOffset() throws IOException; + + protected abstract int doEndOffset() throws IOException; +} Property changes on: lucene/core/src/java/org/apache/lucene/search/PhraseScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/FilterLeafReader.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,14 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Iterator; - import org.apache.lucene.search.CachingWrapperFilter; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import java.io.IOException; +import java.util.Iterator; + /** A FilterLeafReader contains another LeafReader, which it * uses as its basic source of data, possibly transforming the data along the * way or providing additional functionality. The class @@ -220,7 +220,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { return in.docsAndPositions(liveDocs, reuse, flags); } } @@ -267,58 +267,21 @@ } @Override - public long cost() { - return in.cost(); + public int nextPosition() throws IOException { + return in.nextPosition(); } - } - /** Base class for filtering {@link DocsAndPositionsEnum} implementations. */ - public static class FilterDocsAndPositionsEnum extends DocsAndPositionsEnum { - /** The underlying DocsAndPositionsEnum instance. */ - protected final DocsAndPositionsEnum in; - - /** - * Create a new FilterDocsAndPositionsEnum - * @param in the underlying DocsAndPositionsEnum instance. - */ - public FilterDocsAndPositionsEnum(DocsAndPositionsEnum in) { - if (in == null) { - throw new NullPointerException("incoming DocsAndPositionsEnum cannot be null"); - } - this.in = in; - } - @Override - public AttributeSource attributes() { - return in.attributes(); + public int startPosition() throws IOException { + return in.startPosition(); } @Override - public int docID() { - return in.docID(); + public int endPosition() throws IOException { + return in.endPosition(); } @Override - public int freq() throws IOException { - return in.freq(); - } - - @Override - public int nextDoc() throws IOException { - return in.nextDoc(); - } - - @Override - public int advance(int target) throws IOException { - return in.advance(target); - } - - @Override - public int nextPosition() throws IOException { - return in.nextPosition(); - } - - @Override public int startOffset() throws IOException { return in.startOffset(); } @@ -332,7 +295,7 @@ public BytesRef getPayload() throws IOException { return in.getPayload(); } - + @Override public long cost() { return in.cost(); Index: lucene/core/src/java/org/apache/lucene/index/TermContext.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/TermContext.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/TermContext.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,11 @@ * limitations under the License. */ +import org.apache.lucene.util.BytesRef; + import java.io.IOException; import java.util.Arrays; -import org.apache.lucene.util.BytesRef; - /** * Maintains a {@link IndexReader} {@link TermState} view over * {@link IndexReader} instances containing a single term. The Index: lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -21,6 +21,8 @@ import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.util.BytesRef; + /** * Exposes {@link DocsEnum}, merged from {@link DocsEnum} * API of sub-segments. @@ -89,8 +91,38 @@ public int docID() { return doc; } + + @Override + public int nextPosition() throws IOException { + return current.nextPosition(); + } @Override + public int startPosition() throws IOException { + return current.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return current.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return current.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return current.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return current.getPayload(); + } + + @Override public int advance(int target) throws IOException { assert target > doc; while(true) { Index: lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (.../branches/lucene2878) (revision 1643331) @@ -24,7 +24,7 @@ import java.util.Map; import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray; -import org.apache.lucene.util.AttributeSource; // javadocs +import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -256,7 +256,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) { if (liveDocs != null) { throw new IllegalArgumentException("liveDocs must be null"); } @@ -268,7 +268,7 @@ throw new IllegalArgumentException("did not index positions"); } - if (!terms.hasOffsets && (flags & DocsAndPositionsEnum.FLAG_OFFSETS) != 0) { + if (!terms.hasOffsets && (flags & DocsEnum.FLAG_OFFSETS) == DocsEnum.FLAG_OFFSETS) { // Caller wants offsets but we didn't index them; // don't lie: throw new IllegalArgumentException("did not index offsets"); @@ -348,6 +348,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int nextDoc() throws IOException { if (reader.eof()) { if (ended) { @@ -389,7 +419,7 @@ } } - private static class FreqProxDocsAndPositionsEnum extends DocsAndPositionsEnum { + private static class FreqProxDocsAndPositionsEnum extends DocsEnum { final FreqProxTermsWriterPerField terms; final FreqProxPostingsArray postingsArray; @@ -501,6 +531,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() { if (!readOffsets) { throw new IllegalStateException("offsets were not indexed"); Index: lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -401,7 +401,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { MultiDocsAndPositionsEnum docsAndPositionsEnum; // Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) { @@ -452,7 +452,7 @@ } assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length; - final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], flags); + final DocsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], flags); if (subPostings != null) { docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings; Index: lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java (.../trunk) (revision 1643331) +++ lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,10 @@ * limitations under the License. */ +import org.apache.lucene.util.Bits; + import java.io.IOException; -import org.apache.lucene.util.Bits; - import static org.apache.lucene.index.FilterLeafReader.FilterFields; import static org.apache.lucene.index.FilterLeafReader.FilterTerms; import static org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; @@ -120,7 +120,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { if (liveDocs != null) { throw new IllegalArgumentException("liveDocs must be null"); } Index: lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/codecs/perfield/TestPerFieldPostingsFormat2.java (.../branches/lucene2878) (revision 1643331) @@ -16,6 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; @@ -265,7 +266,7 @@ } dir.close(); } - + public void testSameCodecDifferentInstance() throws Exception { Codec codec = new AssertingCodec() { @Override Index: lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (.../branches/lucene2878) (revision 1643331) @@ -26,7 +26,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; @@ -72,7 +72,7 @@ writer.addDocument(doc); IndexReader reader = writer.getReader(); - DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, + DocsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "preanalyzed", new BytesRef("term1")); Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,6 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -31,6 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -46,6 +39,14 @@ import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + public class TestBooleanQuery extends LuceneTestCase { public void testEquality() throws Exception { @@ -235,7 +236,7 @@ Weight weight = s.createNormalizedWeight(q); - Scorer scorer = weight.scorer(s.leafContexts.get(0), null); + Scorer scorer = weight.scorer(s.leafContexts.get(0), DocsEnum.FLAG_FREQS, null); // First pass: just use .nextDoc() to gather all hits final List hits = new ArrayList<>(); @@ -252,7 +253,7 @@ for(int iter2=0;iter2<10;iter2++) { weight = s.createNormalizedWeight(q); - scorer = weight.scorer(s.leafContexts.get(0), null); + scorer = weight.scorer(s.leafContexts.get(0), DocsEnum.FLAG_FREQS, null); if (VERBOSE) { System.out.println(" iter2=" + iter2); Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,68 @@ +package org.apache.lucene.search.posfilter; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; + +// We need to store offsets here, so don't use the following Codecs, which don't +// support them. +@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"}) +public class TestPositionsAndOffsets extends IntervalTestBase { + + protected void addDocs(RandomIndexWriter writer) throws IOException { + FieldType fieldType = new FieldType(TextField.TYPE_NOT_STORED); + fieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + Document doc = new Document(); + doc.add(newField( + "field", + "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some" + + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!", + fieldType)); + writer.addDocument(doc); + } + + public void testTermQueryOffsets() throws IOException { + Query query = new TermQuery(new Term("field", "porridge")); + checkIntervalOffsets(query, searcher, new int[][]{ + { 0, 6, 14, 26, 34, 47, 55, 164, 172, 184, 192 } + }); + } + + public void testBooleanQueryOffsets() throws IOException { + BooleanQuery query = new BooleanQuery(); + query.add(new BooleanClause(new TermQuery(new Term("field", "porridge")), + BooleanClause.Occur.MUST)); + query.add(new BooleanClause(new TermQuery(new Term("field", "nine")), + BooleanClause.Occur.MUST)); + checkIntervalOffsets(query, searcher, new int[][]{ + { 0, 6, 14, 26, 34, 47, 55, 67, 71, 143, 147, 164, 172, 184, 192 } + }); + } + +} \ No newline at end of file Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestPositionsAndOffsets.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/posfilter/TestNestedPositionFilterQueries.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/posfilter/TestNestedPositionFilterQueries.java (.../trunk) (revision 0) +++ lucene/core/src/test/org/apache/lucene/search/posfilter/TestNestedPositionFilterQueries.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,122 @@ +package org.apache.lucene.search.posfilter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; + +import java.io.IOException; + +public class TestNestedPositionFilterQueries extends IntervalTestBase { + + @Override + protected void addDocs(RandomIndexWriter writer) throws IOException { + for (int i = 0; i < docFields.length; i++) { + Document doc = new Document(); + doc.add(newField("field", docFields[i], TextField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + private String[] docFields = { + "w1 w2 w3 w4 w5 w6 w7 w8 w9 w10 w11 w12", //0 + "w1 w3 w4 w5 w6 w7 w8", //1 + "w1 w3 w10 w4 w5 w6 w7 w8", //2 + "w1 w3 w2 w4 w5 w6 w7 w8", //3 + }; + + public void testOrderedDisjunctionQueries() throws IOException { + // Two phrases whose subparts appear in a document, but that do not fulfil the slop + // requirements of the parent IntervalFilterQuery + Query sentence1 = new OrderedNearQuery(0, makeTermQuery("w1"), makeTermQuery("w8"), makeTermQuery("w4")); + Query sentence2 = new OrderedNearQuery(0, makeTermQuery("w3"), makeTermQuery("w7"), makeTermQuery("w6")); + BooleanQuery bq = new BooleanQuery(); + bq.add(sentence1, BooleanClause.Occur.SHOULD); + bq.add(sentence2, BooleanClause.Occur.SHOULD); + checkIntervals(bq, searcher, new int[][]{}); + } + + public void testFilterDisjunctionQuery() throws IOException { + Query near1 = makeTermQuery("w4"); + Query near2 = new OrderedNearQuery(3, makeTermQuery("w1"), makeTermQuery("w10")); + BooleanQuery bq = new BooleanQuery(); + bq.add(near1, BooleanClause.Occur.SHOULD); + bq.add(near2, BooleanClause.Occur.SHOULD); + checkIntervals(bq, searcher, new int[][]{ + { 0, 3, 3 }, + { 1, 2, 2 }, + { 2, 0, 2, 3, 3 }, + { 3, 3, 3 } + }); + } + + // or(w1 pre/2 w2, w1 pre/3 w10) + public void testOrNearNearQuery() throws IOException { + Query near1 = new OrderedNearQuery(2, makeTermQuery("w1"), makeTermQuery("w2")); + Query near2 = new OrderedNearQuery(3, makeTermQuery("w1"), makeTermQuery("w10")); + BooleanQuery bq = new BooleanQuery(); + bq.add(near1, BooleanClause.Occur.SHOULD); + bq.add(near2, BooleanClause.Occur.SHOULD); + checkIntervals(bq, searcher, new int[][]{ + { 0, 0, 1 }, + { 2, 0, 2 }, + { 3, 0, 2 } + }); + } + + // or(w2 within/2 w1, w10 within/3 w1) + public void testUnorderedNearNearQuery() throws IOException { + Query near1 = new UnorderedNearQuery(2, makeTermQuery("w2"), makeTermQuery("w1")); + Query near2 = new UnorderedNearQuery(3, makeTermQuery("w10"), makeTermQuery("w1")); + BooleanQuery bq = new BooleanQuery(); + bq.add(near1, BooleanClause.Occur.SHOULD); + bq.add(near2, BooleanClause.Occur.SHOULD); + checkIntervals(bq, searcher, new int[][]{ + {0, 0, 1}, + {2, 0, 2}, + {3, 0, 2} + }); + } + + // (a pre/2 b) pre/6 (c pre/2 d) + public void testNearNearNearQuery() throws IOException { + Query near1 = new OrderedNearQuery(2, makeTermQuery("w1"), makeTermQuery("w4")); + Query near2 = new OrderedNearQuery(2, makeTermQuery("w10"), makeTermQuery("w12")); + Query near3 = new OrderedNearQuery(6, near1, near2); + checkIntervals(near3, searcher, new int[][]{ + { 0, 0, 11 } + }); + } + + public void testOrNearNearNonExistentQuery() throws IOException { + Query near1 = new OrderedNearQuery(2, makeTermQuery("w1"), makeTermQuery("w12")); + Query near2 = new OrderedNearQuery(2, makeTermQuery("w3"), makeTermQuery("w8")); + BooleanQuery bq = new BooleanQuery(); + bq.add(near1, BooleanClause.Occur.SHOULD); + bq.add(near2, BooleanClause.Occur.SHOULD); + BooleanQuery wrapper = new BooleanQuery(); + wrapper.add(bq, BooleanClause.Occur.MUST); + wrapper.add(makeTermQuery("foo"), BooleanClause.Occur.MUST_NOT); + checkIntervals(wrapper, searcher, new int[][]{}); + } + +} Property changes on: lucene/core/src/test/org/apache/lucene/search/posfilter/TestNestedPositionFilterQueries.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java (.../branches/lucene2878) (revision 1643331) @@ -29,8 +29,9 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; @@ -124,7 +125,7 @@ if (slow) { return new SlowMinShouldMatchScorer(weight, reader, searcher); } else { - return weight.scorer(reader.getContext(), null); + return weight.scorer(reader.getContext(), DocsEnum.FLAG_FREQS, null); } } @@ -315,6 +316,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public int docID() { return currentDoc; } Index: lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,17 +17,27 @@ * limitations under the License. */ -import java.io.*; -import java.util.*; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenFilter; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.document.*; -import org.apache.lucene.index.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.*; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TestUtil; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -35,7 +45,7 @@ * Tests {@link PhraseQuery}. * * @see TestPositionIncrement - */ + */ public class TestPhraseQuery extends LuceneTestCase { /** threshold for comparing floats */ @@ -184,7 +194,7 @@ * slop is the total number of positional moves allowed * to line up a phrase */ - public void testMulipleTerms() throws Exception { + public void testMultipleTerms() throws Exception { query.setSlop(2); query.add(new Term("field", "one")); query.add(new Term("field", "three")); @@ -670,7 +680,7 @@ } } - assertTrue("phrase '" + sb + "' not found; start=" + start, found); + assertTrue("phrase '" + sb + "' not found; start=" + start + ", it=" + i + ", expected doc " + docID, found); } reader.close(); Index: lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestSearchWithThreads.java (.../branches/lucene2878) (revision 1643331) @@ -20,6 +20,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import com.carrotsearch.randomizedtesting.annotations.Seed; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; @@ -26,10 +27,11 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; -import org.apache.lucene.util.LuceneTestCase; @SuppressCodecs({ "SimpleText", "Memory", "Direct" }) +@Seed("12017F5C55C9DD62") public class TestSearchWithThreads extends LuceneTestCase { int NUM_DOCS; final int NUM_SEARCH_THREADS = 5; Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanOr.java (.../branches/lucene2878) (revision 1643331) @@ -15,12 +15,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; -import java.util.concurrent.atomic.AtomicInteger; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -29,6 +27,9 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + public class TestBooleanOr extends LuceneTestCase { private static String FIELD_T = "T"; @@ -182,7 +183,7 @@ Weight w = s.createNormalizedWeight(bq); assertEquals(1, s.getIndexReader().leaves().size()); - BulkScorer scorer = w.bulkScorer(s.getIndexReader().leaves().get(0), false, null); + BulkScorer scorer = w.bulkScorer(s.getIndexReader().leaves().get(0), false, DocsEnum.FLAG_FREQS, null); final FixedBitSet hits = new FixedBitSet(docCount); final AtomicInteger end = new AtomicInteger(); Index: lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (.../branches/lucene2878) (revision 1643331) @@ -72,7 +72,7 @@ assert terms != null; TermsEnum termsEnum = terms.iterator(null); assertTrue(termsEnum.seekExact(new BytesRef("withPayload"))); - DocsAndPositionsEnum de = termsEnum.docsAndPositions(null, null); + DocsEnum de = termsEnum.docsAndPositions(null, null); assertEquals(0, de.nextDoc()); assertEquals(0, de.nextPosition()); assertEquals(new BytesRef("test"), de.getPayload()); @@ -114,7 +114,7 @@ assert terms != null; TermsEnum termsEnum = terms.iterator(null); assertTrue(termsEnum.seekExact(new BytesRef("withPayload"))); - DocsAndPositionsEnum de = termsEnum.docsAndPositions(null, null); + DocsEnum de = termsEnum.docsAndPositions(null, null); assertEquals(0, de.nextDoc()); assertEquals(3, de.nextPosition()); assertEquals(new BytesRef("test"), de.getPayload()); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (.../branches/lucene2878) (revision 1643331) @@ -833,7 +833,7 @@ Terms tpv = r.getTermVectors(0).terms("field"); TermsEnum termsEnum = tpv.iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); Index: lucene/core/src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (.../branches/lucene2878) (revision 1643331) @@ -17,18 +17,16 @@ * limitations under the License. */ -import java.io.IOException; -import java.io.StringReader; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.lucene.analysis.*; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.CannedTokenStream; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; @@ -35,11 +33,20 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.io.StringReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + public class TestPayloads extends LuceneTestCase { // Simple tests to test the Payload class @@ -183,7 +190,7 @@ byte[] verifyPayloadData = new byte[payloadDataLength]; offset = 0; - DocsAndPositionsEnum[] tps = new DocsAndPositionsEnum[numTerms]; + DocsEnum[] tps = new DocsEnum[numTerms]; for (int i = 0; i < numTerms; i++) { tps[i] = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), @@ -214,7 +221,7 @@ /* * test lazy skipping */ - DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader, + DocsEnum tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), terms[0].field(), new BytesRef(terms[0].text())); @@ -481,7 +488,7 @@ IndexReader reader = DirectoryReader.open(dir); TermsEnum terms = MultiFields.getFields(reader).terms(field).iterator(null); Bits liveDocs = MultiFields.getLiveDocs(reader); - DocsAndPositionsEnum tp = null; + DocsEnum tp = null; while (terms.next() != null) { String termText = terms.term().utf8ToString(); tp = terms.docsAndPositions(liveDocs, tp); @@ -604,7 +611,7 @@ writer.addDocument(doc); DirectoryReader reader = writer.getReader(); LeafReader sr = SlowCompositeReaderWrapper.wrap(reader); - DocsAndPositionsEnum de = sr.termPositionsEnum(new Term("field", "withPayload")); + DocsEnum de = sr.termPositionsEnum(new Term("field", "withPayload")); de.nextDoc(); de.nextPosition(); assertEquals(new BytesRef("test"), de.getPayload()); @@ -638,7 +645,7 @@ writer.addDocument(doc); DirectoryReader reader = writer.getReader(); SegmentReader sr = getOnlySegmentReader(reader); - DocsAndPositionsEnum de = sr.termPositionsEnum(new Term("field", "withPayload")); + DocsEnum de = sr.termPositionsEnum(new Term("field", "withPayload")); de.nextDoc(); de.nextPosition(); assertEquals(new BytesRef("test"), de.getPayload()); Index: lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (.../branches/lucene2878) (revision 1643331) @@ -154,7 +154,7 @@ writer.close(); IndexReader reader = DirectoryReader.open(directory); - DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader, + DocsEnum tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), this.field, new BytesRef("b")); Index: lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestFilterLeafReader.java (.../branches/lucene2878) (revision 1643331) @@ -18,10 +18,6 @@ */ -import java.io.IOException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -32,6 +28,10 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; + public class TestFilterLeafReader extends LuceneTestCase { private static class TestReader extends FilterLeafReader { @@ -76,14 +76,14 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { - return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, flags)); + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsEnum) reuse).in, flags)); } } /** Filter that only returns odd numbered documents. */ - private static class TestPositions extends FilterDocsAndPositionsEnum { - public TestPositions(DocsAndPositionsEnum in) { + private static class TestPositions extends FilterDocsEnum { + public TestPositions(DocsEnum in) { super(in); } @@ -151,7 +151,7 @@ assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one"))); - DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader), null); + DocsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader), null); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue((positions.docID() % 2) == 1); } @@ -189,7 +189,6 @@ checkOverrideMethods(FilterLeafReader.FilterTerms.class); checkOverrideMethods(FilterLeafReader.FilterTermsEnum.class); checkOverrideMethods(FilterLeafReader.FilterDocsEnum.class); - checkOverrideMethods(FilterLeafReader.FilterDocsAndPositionsEnum.class); } } Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (.../branches/lucene2878) (revision 1643331) @@ -633,8 +633,8 @@ while(enum1.next() != null) { assertEquals("Different terms", enum1.term(), enum2.next()); - DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null); - DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null); + DocsEnum tp1 = enum1.docsAndPositions(liveDocs, null); + DocsEnum tp2 = enum2.docsAndPositions(liveDocs, null); while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java (.../branches/lucene2878) (revision 1643331) @@ -16,10 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -32,6 +28,10 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; + public class TestDocsAndPositions extends LuceneTestCase { private String fieldName; @@ -42,7 +42,7 @@ } /** - * Simple testcase for {@link DocsAndPositionsEnum} + * Simple testcase for {@link DocsEnum} */ public void testPositionsSimple() throws IOException { Directory directory = newDirectory(); @@ -65,7 +65,7 @@ BytesRef bytes = new BytesRef("1"); IndexReaderContext topReaderContext = reader.getContext(); for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) { - DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( + DocsEnum docsAndPosEnum = getDocsAndPositions( leafReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); if (leafReaderContext.reader().maxDoc() == 0) { @@ -90,7 +90,7 @@ directory.close(); } - public DocsAndPositionsEnum getDocsAndPositions(LeafReader reader, + public DocsEnum getDocsAndPositions(LeafReader reader, BytesRef bytes, Bits liveDocs) throws IOException { Terms terms = reader.terms(fieldName); if (terms != null) { @@ -148,7 +148,7 @@ BytesRef bytes = new BytesRef("" + term); IndexReaderContext topReaderContext = reader.getContext(); for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) { - DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( + DocsEnum docsAndPosEnum = getDocsAndPositions( leafReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); int initDoc = 0; @@ -303,7 +303,7 @@ IndexReaderContext topReaderContext = reader.getContext(); for (LeafReaderContext leafReaderContext : topReaderContext.leaves()) { - DocsAndPositionsEnum docsAndPosEnum = getDocsAndPositions( + DocsEnum docsAndPosEnum = getDocsAndPositions( leafReaderContext.reader(), bytes, null); assertNotNull(docsAndPosEnum); @@ -361,7 +361,7 @@ writer.addDocument(doc); DirectoryReader reader = writer.getReader(); LeafReader r = getOnlySegmentReader(reader); - DocsAndPositionsEnum disi = r.termPositionsEnum(new Term("foo", "bar")); + DocsEnum disi = r.termPositionsEnum(new Term("foo", "bar")); int docid = disi.docID(); assertEquals(-1, docid); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (.../branches/lucene2878) (revision 1643331) @@ -247,7 +247,7 @@ assertNotNull(vector); assertEquals(testTerms.length, vector.size()); TermsEnum termsEnum = vector.iterator(null); - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; for (int i = 0; i < testTerms.length; i++) { final BytesRef text = termsEnum.next(); assertNotNull(text); @@ -304,7 +304,7 @@ TermsEnum termsEnum = vector.iterator(null); assertNotNull(termsEnum); assertEquals(testTerms.length, vector.size()); - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; for (int i = 0; i < testTerms.length; i++) { final BytesRef text = termsEnum.next(); assertNotNull(text); Index: lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (.../branches/lucene2878) (revision 1643331) @@ -17,9 +17,6 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Random; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -32,6 +29,9 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.util.Random; + /** * Compares one codec against another */ Index: lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (.../branches/lucene2878) (revision 1643331) @@ -167,7 +167,7 @@ System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1); } - final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term)); + final DocsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term)); int docID = -1; while(docID < DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (.../branches/lucene2878) (revision 1643331) @@ -396,7 +396,7 @@ Fields tv1 = r1.getTermVectors(id1); System.out.println(" d1=" + tv1); if (tv1 != null) { - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; DocsEnum dEnum = null; for (String field : tv1) { System.out.println(" " + field + ":"); @@ -428,7 +428,7 @@ Fields tv2 = r2.getTermVectors(id2); System.out.println(" d2=" + tv2); if (tv2 != null) { - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; DocsEnum dEnum = null; for (String field : tv2) { System.out.println(" " + field + ":"); @@ -616,8 +616,8 @@ assertNotNull(terms2); TermsEnum termsEnum2 = terms2.iterator(null); - DocsAndPositionsEnum dpEnum1 = null; - DocsAndPositionsEnum dpEnum2 = null; + DocsEnum dpEnum1 = null; + DocsEnum dpEnum2 = null; DocsEnum dEnum1 = null; DocsEnum dEnum2 = null; Index: lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (.../branches/lucene2878) (revision 1643331) @@ -84,7 +84,7 @@ for (int i = 0; i < 2; i++) { counter = 0; - DocsAndPositionsEnum tp = reader.termPositionsEnum(term); + DocsEnum tp = reader.termPositionsEnum(term); checkSkipTo(tp, 14, 185); // no skips checkSkipTo(tp, 17, 190); // one skip on level 0 checkSkipTo(tp, 287, 200); // one skip on level 1, two on level 0 @@ -95,7 +95,7 @@ } } - public void checkSkipTo(DocsAndPositionsEnum tp, int target, int maxCounter) throws IOException { + public void checkSkipTo(DocsEnum tp, int target, int maxCounter) throws IOException { tp.advance(target); if (maxCounter < counter) { fail("Too many bytes read: " + counter + " vs " + maxCounter); Index: lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java (.../trunk) (revision 1643331) +++ lucene/join/src/java/org/apache/lucene/search/join/FakeScorer.java (.../branches/lucene2878) (revision 1643331) @@ -1,75 +0,0 @@ -package org.apache.lucene.search.join; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Collection; - -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; - -/** Passed to {@link LeafCollector#setScorer} during join collection. */ -final class FakeScorer extends Scorer { - float score; - int doc = -1; - int freq = 1; - - public FakeScorer() { - super(null); - } - - @Override - public int advance(int target) { - throw new UnsupportedOperationException("FakeScorer doesn't support advance(int)"); - } - - @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - throw new UnsupportedOperationException("FakeScorer doesn't support freq()"); - } - - @Override - public int nextDoc() { - throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()"); - } - - @Override - public float score() { - return score; - } - - @Override - public long cost() { - return 1; - } - - @Override - public Weight getWeight() { - throw new UnsupportedOperationException(); - } - - @Override - public Collection getChildren() { - throw new UnsupportedOperationException(); - } -} Index: lucene/replicator =================================================================== --- lucene/replicator (.../trunk) (revision 1643331) +++ lucene/replicator (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/replicator ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/replicator:r1638800-1642144 Index: lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (.../trunk) (revision 1643331) +++ lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (.../branches/lucene2878) (revision 1643331) @@ -22,7 +22,7 @@ import java.util.LinkedList; import java.util.Set; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; @@ -93,7 +93,7 @@ final CharsRefBuilder spare = new CharsRefBuilder(); final TermsEnum termsEnum = vector.iterator(null); - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; BytesRef text; int numDocs = reader.maxDoc(); Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (.../branches/lucene2878) (revision 1643331) @@ -68,7 +68,7 @@ // Token "" occurred once assertEquals(1, termsEnum.totalTermFreq()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(8, dpEnum.startOffset()); @@ -117,7 +117,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -152,7 +152,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -192,7 +192,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -227,7 +227,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -263,7 +263,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); @@ -307,7 +307,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -349,7 +349,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (.../branches/lucene2878) (revision 1643331) @@ -145,7 +145,7 @@ assertTrue(termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader, + DocsEnum positions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), DocHelper.TEXT_FIELD_1_KEY, new BytesRef("field")); Index: lucene/core/src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDoc.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestDoc.java (.../branches/lucene2878) (revision 1643331) @@ -261,7 +261,7 @@ out.print(" term=" + field + ":" + tis.term()); out.println(" DF=" + tis.docFreq()); - DocsAndPositionsEnum positions = tis.docsAndPositions(reader.getLiveDocs(), null); + DocsEnum positions = tis.docsAndPositions(reader.getLiveDocs(), null); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { out.print(" doc=" + positions.docID()); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (.../branches/lucene2878) (revision 1643331) @@ -332,7 +332,7 @@ TermsEnum termsEnum = tfv.iterator(null); assertEquals(new BytesRef(""+counter), termsEnum.next()); assertEquals(1, termsEnum.totalTermFreq()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); + DocsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(1, dpEnum.nextPosition()); Index: lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java =================================================================== --- lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java (.../trunk) (revision 1643331) +++ lucene/highlighter/src/test/org/apache/lucene/search/highlight/custom/HighlightCustomQueryTest.java (.../branches/lucene2878) (revision 1643331) @@ -16,9 +16,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; -import java.util.Map; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenFilter; import org.apache.lucene.analysis.MockTokenizer; @@ -36,6 +33,9 @@ import org.apache.lucene.search.highlight.WeightedSpanTermExtractor; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; +import java.util.Map; + /** * Tests the extensibility of {@link WeightedSpanTermExtractor} and * {@link QueryScorer} in a user defined package Index: lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java (.../trunk) (revision 1643331) +++ lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/MultiTermHighlighting.java (.../branches/lucene2878) (revision 1643331) @@ -26,7 +26,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Term; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.BooleanClause; @@ -47,10 +47,10 @@ import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.UnicodeUtil; import org.apache.lucene.util.automaton.Automata; -import org.apache.lucene.util.automaton.Operations; +import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.LevenshteinAutomata; -import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.Operations; /** * Support for highlighting multiterm queries in PostingsHighlighter. @@ -197,7 +197,7 @@ *

    * This is solely used internally by PostingsHighlighter: DO NOT USE THIS METHOD! */ - static DocsAndPositionsEnum getDocsEnum(final TokenStream ts, final CharacterRunAutomaton[] matchers) throws IOException { + static DocsEnum getDocsEnum(final TokenStream ts, final CharacterRunAutomaton[] matchers) throws IOException { final CharTermAttribute charTermAtt = ts.addAttribute(CharTermAttribute.class); final OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); ts.reset(); @@ -207,7 +207,7 @@ // would only serve to make this method less bogus. // instead, we always return freq() = Integer.MAX_VALUE and let PH terminate based on offset... - return new DocsAndPositionsEnum() { + return new DocsEnum() { int currentDoc = -1; int currentMatch = -1; int currentStartOffset = -1; @@ -237,8 +237,20 @@ currentStartOffset = currentEndOffset = Integer.MAX_VALUE; return Integer.MAX_VALUE; } - + @Override + public int startPosition() throws IOException { + if (currentStartOffset < Integer.MAX_VALUE) + return 0; + return NO_MORE_POSITIONS; + } + + @Override + public int endPosition() throws IOException { + return startPosition(); + } + + @Override public int freq() throws IOException { return Integer.MAX_VALUE; // lie } Index: lucene/core/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (.../trunk) (revision 1643331) +++ lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (.../branches/lucene2878) (revision 1643331) @@ -343,7 +343,7 @@ assertTrue(doc != DocIdSetIterator.NO_MORE_DOCS); assertEquals(docs[i], doc); if (doPos) { - this.verifyPositions(positions[i], ((DocsAndPositionsEnum) docsEnum)); + this.verifyPositions(positions[i], docsEnum); } } assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsEnum.nextDoc()); @@ -351,7 +351,7 @@ byte[] data = new byte[10]; - private void verifyPositions(final PositionData[] positions, final DocsAndPositionsEnum posEnum) throws Throwable { + private void verifyPositions(final PositionData[] positions, final DocsEnum posEnum) throws Throwable { for(int i=0;itest test", frags[0]); + close(); + } + + public void testSeveralSnippets() throws Exception { + String input = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + + "Let us see what happens to long in this case."; + String gold = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + + "Let us see what happens to long in this case."; + insertDocs(input); + String frags[] = doSearch(termQuery("long"), input.length()); + assertEquals(gold, frags[0]); + close(); + } + + public void testBooleanAnd() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(termQuery("This"), Occur.MUST)); + bq.add(new BooleanClause(termQuery("test"), Occur.MUST)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testConstantScore() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(termQuery("This"), Occur.MUST)); + bq.add(new BooleanClause(termQuery("test"), Occur.MUST)); + String frags[] = doSearch(new ConstantScoreQuery(bq)); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testBooleanAndOtherOrder() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.MUST)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testBooleanOr() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testSingleMatchScorer() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "notoccurringterm")), + Occur.SHOULD)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testBooleanNrShouldMatch() throws Exception { + insertDocs("a b c d e f g h i"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "a")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "b")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "no")), Occur.SHOULD)); + + // This generates a ConjunctionSumScorer + bq.setMinimumNumberShouldMatch(2); + String frags[] = doSearch(bq); + assertEquals("a b c d e f g h i", frags[0]); + + // This generates no scorer + bq.setMinimumNumberShouldMatch(3); + frags = doSearch(bq); + assertNull(frags); + + // This generates a DisjunctionSumScorer + bq.setMinimumNumberShouldMatch(2); + bq.add(new BooleanClause(new TermQuery(new Term(F, "c")), Occur.SHOULD)); + frags = doSearch(bq); + assertEquals("a b c d e f g h i", frags[0]); + close(); + } + + public void testPhrase() throws Exception { + insertDocs("is it that this is a test, is it"); + PhraseQuery pq = new PhraseQuery(); + pq.add(new Term(F, "is")); + pq.add(new Term(F, "a")); + String frags[] = doSearch(pq); + // make sure we highlight the phrase, and not the terms outside the phrase + assertEquals("is it that this is a test, is it", frags[0]); + close(); + } + + /* + * Failing ... PhraseQuery scorer needs positions()? + */ + //@Ignore + public void testPhraseOriginal() throws Exception { + insertDocs("This is a test"); + PhraseQuery pq = new PhraseQuery(); + pq.add(new Term(F, "a")); + pq.add(new Term(F, "test")); + String frags[] = doSearch(pq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testNestedBoolean() throws Exception { + insertDocs("This is a test"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + BooleanQuery bq2 = new BooleanQuery(); + bq2.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD)); + bq2.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.SHOULD)); + bq.add(new BooleanClause(bq2, Occur.SHOULD)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testWildcard() throws Exception { + insertDocs("This is a test"); + String frags[] = doSearch(new WildcardQuery(new Term(F, "t*t"))); + assertEquals("This is a test", frags[0]); + close(); + } + + public void testMixedBooleanNot() throws Exception { + insertDocs("this is a test", "that is an elephant"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "that")), Occur.MUST_NOT)); + String frags[] = doSearch(bq); + assertEquals("this is a test", frags[0]); + close(); + } + + public void testMixedBooleanShould() throws Exception { + insertDocs("this is a test", "that is an elephant", "the other was a rhinoceros"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + String frags[] = doSearch(bq, 50, 0); + assertEquals("this is a test", frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("that is an elephant", frags[0]); + + bq.add(new BooleanClause(new TermQuery(new Term(F, "rhinoceros")), Occur.SHOULD)); + frags = doSearch(bq, 50, 0); + assertEquals("this is a test", frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("that is an elephant", frags[0]); + close(); + } + + public void testMultipleDocumentsAnd() throws Exception { + insertDocs("This document has no matches", PORRIDGE_VERSE, + "This document has some Pease porridge in it"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")), Occur.MUST)); + String frags[] = doSearch(bq, 50, 0); + assertEquals( + "Pease porridge hot! Pease porridge cold! Pease", + frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("This document has some Pease porridge in it", + frags[0]); + close(); + } + + + public void testMultipleDocumentsOr() throws Exception { + insertDocs("This document has no matches", PORRIDGE_VERSE, + "This document has some Pease porridge in it"); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")), + Occur.SHOULD)); + String frags[] = doSearch(bq, 50, 0); + assertEquals( + "Pease porridge hot! Pease porridge cold! Pease", + frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("This document has some Pease porridge in it", + frags[0]); + close(); + } + + public void testBrouwerianQuery() throws Exception { + + insertDocs("the quick brown fox jumps over the lazy dog with the quick orange fox"); + + OrderedNearQuery query = new OrderedNearQuery(1, + new TermQuery(new Term(F, "the")), new TermQuery(new Term(F, "quick")), new TermQuery(new Term(F, "fox"))); + + assertEquals(getHighlight(query), + "the quick brown fox jumps over the lazy dog with the quick orange fox"); + + NonOverlappingQuery bq = new NonOverlappingQuery(query, new TermQuery(new Term(F, "orange"))); + + assertEquals(getHighlight(bq), + "the quick brown fox jumps over the lazy dog with the quick orange fox"); + + close(); + } + + //@Ignore("not implemented yet - unsupported") + public void testMultiPhraseQuery() throws Exception { + MultiPhraseQuery query = new MultiPhraseQuery(); + insertDocs("pease porridge hot but not too hot or otherwise pease porridge cold"); + + query.add(terms(F, "pease"), 0); + query.add(terms(F, "porridge"), 1); + query.add(terms(F, "hot", "cold"), 2); + query.setSlop(1); + + String[] frags = doSearch(query, Integer.MAX_VALUE); + assertEquals("pease porridge hot but not too hot or otherwise pease porridge cold", frags[0]); + + close(); + } + + //@Ignore("not implemented yet - unsupported") + public void testMultiPhraseQueryCollisions() throws Exception { + MultiPhraseQuery query = new MultiPhraseQuery(); + insertDocs("pease porridge hot not too hot or otherwise pease porridge porridge"); + + query.add(terms(F, "pease"), 0); + query.add(terms(F, "porridge"), 1); + query.add(terms(F, "coldasice", "porridge" ), 2); + query.setSlop(1); + + String[] frags = doSearch(query, Integer.MAX_VALUE); + assertEquals("pease porridge hot not too hot or otherwise pease porridge porridge", frags[0]); + + close(); + } + + public void testNearPhraseQuery() throws Exception { + + insertDocs("pease porridge rather hot and pease porridge fairly cold"); + + Query firstQ = new OrderedNearQuery(4, termQuery("pease"), termQuery("porridge"), termQuery("hot")); + { + String frags[] = doSearch(firstQ, Integer.MAX_VALUE); + assertEquals("pease porridge rather hot and pease porridge fairly cold", frags[0]); + } + + // near.3(near.4(pease, porridge, hot), near.4(pease, porridge, cold)) + Query q = new OrderedNearQuery(3, + firstQ, + new OrderedNearQuery(4, termQuery("pease"), termQuery("porridge"), termQuery("cold"))); + + String frags[] = doSearch(q, Integer.MAX_VALUE); + assertEquals("pease porridge rather hot and pease porridge fairly cold", + frags[0]); + + close(); + } + + private Term[] terms(String field, String...tokens) { + Term[] terms = new Term[tokens.length]; + for (int i = 0; i < tokens.length; i++) { + terms[i] = new Term(field, tokens[i]); + } + return terms; + } + + public void testSloppyPhraseQuery() throws Exception { + assertSloppyPhrase( "a c e b d e f a b", "a c e b d e f a b", 2, "a", "b"); + assertSloppyPhrase( "a b c d a b c d e f", "a b c d a b c d e f", 2, "c", "a"); + assertSloppyPhrase( "Y A X B A", "Y A X B A", 2, "X", "A", "A"); + assertSloppyPhrase( "X A X B A","X A X B A", 2, "X", "A", "A"); // non overlapping minmal!! + + assertSloppyPhrase( "A A A X",null, 2, "X", "A", "A"); + assertSloppyPhrase( "A A X A", "A A X A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A X A Y B A", "A A X A Y B A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A X", null, 2, "X", "A", "A"); + assertSloppyPhrase( "A X A", null, 1, "X", "A", "A"); + + assertSloppyPhrase( "A X B A", "A X B A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A X A X B A X B B A A X B A A", "A A X A X B A X B B A A X B A A", 2, "X", "A", "A"); + + assertSloppyPhrase( "A A X A X B A", "A A X A X B A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A Y A X B A", "A A Y A X B A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A Y A X B A A", "A A Y A X B A A", 2, "X", "A", "A"); + assertSloppyPhrase( "A A X A Y B A", null , 1, "X", "A", "A"); + close(); + } + + + private void assertSloppyPhrase(String doc, String expected, int slop, String...query) throws Exception { + insertDocs(doc); + PhraseQuery pq = new PhraseQuery(); + for (String string : query) { + pq.add(new Term(F, string)); + } + + pq.setSlop(slop); +// System.out.println(doc); + String[] frags = doSearch(pq, 50); + if (expected == null) { + assertNull(frags != null ? frags[0] : "", frags); + } else { + assertEquals(expected, frags[0]); + } + } + +} Property changes on: lucene/highlighter/src/test/org/apache/lucene/search/highlight/positions/IntervalHighlighterTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (.../trunk) (revision 1643331) +++ lucene/highlighter/src/java/org/apache/lucene/search/postingshighlight/PostingsHighlighter.java (.../branches/lucene2878) (revision 1643331) @@ -31,7 +31,7 @@ import java.util.TreeSet; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; @@ -455,7 +455,7 @@ private Map highlightField(String field, String contents[], BreakIterator bi, BytesRef terms[], int[] docids, List leaves, int maxPassages, Query query) throws IOException { Map highlights = new HashMap<>(); - + PassageFormatter fieldFormatter = getFormatter(field); if (fieldFormatter == null) { throw new NullPointerException("PassageFormatter cannot be null"); @@ -477,7 +477,7 @@ // we are processing in increasing docid order, so we only need to reinitialize stuff on segment changes // otherwise, we will just advance() existing enums to the new document in the same segment. - DocsAndPositionsEnum postings[] = null; + DocsEnum postings[] = null; TermsEnum termsEnum = null; int lastLeaf = -1; @@ -499,7 +499,7 @@ Terms t = r.terms(field); if (t != null) { termsEnum = t.iterator(null); - postings = new DocsAndPositionsEnum[terms.length]; + postings = new DocsEnum[terms.length]; } } if (termsEnum == null) { @@ -508,7 +508,7 @@ // if there are multi-term matches, we have to initialize the "fake" enum for each document if (automata.length > 0) { - DocsAndPositionsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata); + DocsEnum dp = MultiTermHighlighting.getDocsEnum(analyzer.tokenStream(field, content), automata); dp.advance(doc - subContext.docBase); postings[terms.length-1] = dp; // last term is the multiterm matcher } @@ -534,7 +534,7 @@ // we can intersect these with the postings lists via BreakIterator.preceding(offset),s // score each sentence as norm(sentenceStartOffset) * sum(weight * tf(freq)) private Passage[] highlightDoc(String field, BytesRef terms[], int contentLength, BreakIterator bi, int doc, - TermsEnum termsEnum, DocsAndPositionsEnum[] postings, int n) throws IOException { + TermsEnum termsEnum, DocsEnum[] postings, int n) throws IOException { PassageScorer scorer = getScorer(field); if (scorer == null) { throw new NullPointerException("PassageScorer cannot be null"); @@ -543,7 +543,7 @@ float weights[] = new float[terms.length]; // initialize postings for (int i = 0; i < terms.length; i++) { - DocsAndPositionsEnum de = postings[i]; + DocsEnum de = postings[i]; int pDoc; if (de == EMPTY) { continue; @@ -552,7 +552,7 @@ if (!termsEnum.seekExact(terms[i])) { continue; // term not found } - de = postings[i] = termsEnum.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_OFFSETS); + de = postings[i] = termsEnum.docsAndPositions(null, null, DocsEnum.FLAG_OFFSETS); if (de == null) { // no positions available throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight"); @@ -590,7 +590,7 @@ OffsetsEnum off; while ((off = pq.poll()) != null) { - final DocsAndPositionsEnum dp = off.dp; + final DocsEnum dp = off.dp; int start = dp.startOffset(); if (start == -1) { throw new IllegalArgumentException("field '" + field + "' was indexed without offsets, cannot highlight"); @@ -698,11 +698,11 @@ } private static class OffsetsEnum implements Comparable { - DocsAndPositionsEnum dp; + DocsEnum dp; int pos; int id; - OffsetsEnum(DocsAndPositionsEnum dp, int id) throws IOException { + OffsetsEnum(DocsEnum dp, int id) throws IOException { this.dp = dp; this.id = id; this.pos = 1; @@ -724,18 +724,28 @@ } } - private static final DocsAndPositionsEnum EMPTY = new DocsAndPositionsEnum() { + private static final DocsEnum EMPTY = new DocsEnum() { @Override - public int nextPosition() throws IOException { return 0; } + public int nextPosition() throws IOException { return NO_MORE_POSITIONS; } @Override - public int startOffset() throws IOException { return Integer.MAX_VALUE; } + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } @Override - public int endOffset() throws IOException { return Integer.MAX_VALUE; } + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } @Override + public int startOffset() throws IOException { return NO_MORE_POSITIONS; } + + @Override + public int endOffset() throws IOException { return NO_MORE_POSITIONS; } + + @Override public BytesRef getPayload() throws IOException { return null; } @Override Index: lucene/core =================================================================== --- lucene/core (.../trunk) (revision 1643331) +++ lucene/core (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/core ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/core:r1638800-1643324 Index: lucene/SYSTEM_REQUIREMENTS.txt =================================================================== --- lucene/SYSTEM_REQUIREMENTS.txt (.../trunk) (revision 1643331) +++ lucene/SYSTEM_REQUIREMENTS.txt (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/SYSTEM_REQUIREMENTS.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/SYSTEM_REQUIREMENTS.txt:r1638800-1642144 Index: lucene/ivy-versions.properties =================================================================== --- lucene/ivy-versions.properties (.../trunk) (revision 1643331) +++ lucene/ivy-versions.properties (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/ivy-versions.properties ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/ivy-versions.properties:r1638800-1642144 Index: lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsEnum.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsEnum.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,8 +17,11 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.index.DocsEnum; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; class SingleDocsEnum extends DocsEnum { @@ -68,4 +71,34 @@ public int freq() { return 1; } + + @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } } Index: lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,7 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; @@ -40,6 +33,13 @@ import org.apache.lucene.util.automaton.Operations; import org.apache.lucene.util.automaton.Transition; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + import static org.apache.lucene.util.automaton.Operations.DEFAULT_MAX_DETERMINIZED_STATES; // TODO @@ -325,7 +325,7 @@ static class EnumAndScorer { public final int termID; - public final DocsAndPositionsEnum posEnum; + public final DocsEnum posEnum; // How many positions left in the current document: public int posLeft; @@ -333,7 +333,7 @@ // Current position public int pos; - public EnumAndScorer(int termID, DocsAndPositionsEnum posEnum) { + public EnumAndScorer(int termID, DocsEnum posEnum) { this.termID = termID; this.posEnum = posEnum; } @@ -385,7 +385,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { // Initialize the enums; null for a given slot means that term didn't appear in this reader EnumAndScorer[] enums = new EnumAndScorer[idToTerm.size()]; Index: lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (.../trunk) (revision 1643331) +++ lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,18 +17,33 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; +import java.util.Queue; + +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexWriter; // javadocs -import org.apache.lucene.search.*; +import org.apache.lucene.search.FakeScorer; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.FieldValueHitQueue; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreCachingWrappingScorer; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Scorer.ChildScorer; +import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.search.grouping.GroupDocs; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.util.ArrayUtil; -import java.io.IOException; -import java.util.*; - /** Collects parent document hits for a Query containing one more more * BlockJoinQuery clauses, sorted by the * specified parent Sort. Note that this cannot perform Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (.../trunk) (revision 0) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,137 @@ +package org.apache.lucene.search.highlight; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CollectionUtil; + +/** + * TokenStream created from a term vector field. + */ +public final class TokenStreamFromTermPositionVector extends TokenStream { + + private final List positionedTokens = new ArrayList<>(); + + private Iterator tokensAtCurrentPosition; + + private CharTermAttribute termAttribute; + + private PositionIncrementAttribute positionIncrementAttribute; + + private OffsetAttribute offsetAttribute; + + private PayloadAttribute payloadAttribute; + + /** + * Constructor. + * + * @param vector Terms that contains the data for + * creating the TokenStream. Must have positions and offsets. + */ + public TokenStreamFromTermPositionVector( + final Terms vector) throws IOException { + termAttribute = addAttribute(CharTermAttribute.class); + positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); + offsetAttribute = addAttribute(OffsetAttribute.class); + payloadAttribute = addAttribute(PayloadAttribute.class); + final boolean hasOffsets = vector.hasOffsets(); + final boolean hasPayloads = vector.hasPayloads(); + final TermsEnum termsEnum = vector.iterator(null); + BytesRef text; + DocsEnum dpEnum = null; + while((text = termsEnum.next()) != null) { + dpEnum = termsEnum.docsAndPositions(null, dpEnum); + assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlier + dpEnum.nextDoc(); + final int freq = dpEnum.freq(); + for (int j = 0; j < freq; j++) { + int pos = dpEnum.nextPosition(); + Token token; + if (hasOffsets) { + token = new Token(text.utf8ToString(), + dpEnum.startOffset(), + dpEnum.endOffset()); + } else { + token = new Token(); + token.setEmpty().append(text.utf8ToString()); + } + if (hasPayloads) { + // Must make a deep copy of the returned payload, + // since D&PEnum API is allowed to re-use on every + // call: + token.setPayload(BytesRef.deepCopyOf(dpEnum.getPayload())); + } + + // Yes - this is the position, not the increment! This is for + // sorting. This value + // will be corrected before use. + token.setPositionIncrement(pos); + this.positionedTokens.add(token); + } + } + CollectionUtil.timSort(this.positionedTokens, tokenComparator); + int lastPosition = -1; + for (final Token token : this.positionedTokens) { + int thisPosition = token.getPositionIncrement(); + token.setPositionIncrement(thisPosition - lastPosition); + lastPosition = thisPosition; + } + this.tokensAtCurrentPosition = this.positionedTokens.iterator(); + } + + private static final Comparator tokenComparator = new Comparator() { + @Override + public int compare(final Token o1, final Token o2) { + return o1.getPositionIncrement() - o2.getPositionIncrement(); + } + }; + + @Override + public boolean incrementToken() { + if (this.tokensAtCurrentPosition.hasNext()) { + final Token next = this.tokensAtCurrentPosition.next(); + clearAttributes(); + termAttribute.setEmpty().append(next); + positionIncrementAttribute.setPositionIncrement(next + .getPositionIncrement()); + offsetAttribute.setOffset(next.startOffset(), next.endOffset()); + payloadAttribute.setPayload(next.getPayload()); + return true; + } + return false; + } + + @Override + public void reset() { + this.tokensAtCurrentPosition = this.positionedTokens.iterator(); + } +} Property changes on: lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/IntervalTokenStream.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/IntervalTokenStream.java (.../trunk) (revision 0) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/IntervalTokenStream.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,79 @@ +package org.apache.lucene.search.highlight.positions; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.search.posfilter.Interval; + +/** + * A TokenStream constructed from a stream of positions and their offsets. + * The document is segmented into tokens at the start and end offset of each interval. If + * an interval overlaps the previous returned interval, it is skipped. + * + * TODO: abstract the dependency on the current PositionOffsetMapper impl; + * allow for implementations of position->offset maps that don't rely on term vectors. + * + * @lucene.experimental + */ +public class IntervalTokenStream extends TokenStream { + + //this tokenizer generates four attributes: + // term, offset, positionIncrement? and type? + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + //private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class); + private final String text; + private final Interval[] positions; + + // the index of the current position interval + private int index = -1; + + // last end offset returned (to avoid overlaps) + private int lastEndOffset = -1; + + public IntervalTokenStream (String text, Interval[] positions) { + this.text = text; + this.positions = positions; + } + + @Override + public final boolean incrementToken() throws IOException { + int b = -1, e = -1; + while (b <= lastEndOffset) { + if (++index >= positions.length) + return false; + if (positions[index] == null) + return false; + b = positions[index].offsetBegin; + e = positions[index].offsetEnd; + assert b >= 0; + } + lastEndOffset = e; + termAtt.append(text, b, e); + offsetAtt.setOffset(b, e); + posIncrAtt.setPositionIncrement(1); + return true; + } + +} Property changes on: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/IntervalTokenStream.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionPostingsReader.java (.../branches/lucene2878) (revision 1643331) @@ -22,7 +22,6 @@ import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.PostingsReaderBase; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; @@ -78,8 +77,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, - DocsAndPositionsEnum reuse, int flags) { + public DocsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, + DocsEnum reuse, int flags) { SingleDocsAndPositionsEnum posEnum; if (reuse instanceof SingleDocsAndPositionsEnum) { Index: lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java (.../trunk) (revision 1643331) +++ lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java (.../branches/lucene2878) (revision 1643331) @@ -23,6 +23,7 @@ import java.util.Locale; import java.util.Set; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; @@ -39,6 +40,7 @@ import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; /** * This query requires that you index @@ -160,9 +162,9 @@ // NOTE: acceptDocs applies (and is checked) only in the // parent document space @Override - public Scorer scorer(LeafReaderContext readerContext, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext readerContext, int flags, Bits acceptDocs) throws IOException { - final Scorer childScorer = childWeight.scorer(readerContext, readerContext.reader().getLiveDocs()); + final Scorer childScorer = childWeight.scorer(readerContext, flags, readerContext.reader().getLiveDocs()); if (childScorer == null) { // No matches return null; @@ -188,7 +190,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - BlockJoinScorer scorer = (BlockJoinScorer) scorer(context, context.reader().getLiveDocs()); + BlockJoinScorer scorer = (BlockJoinScorer) scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); if (scorer != null && scorer.advance(doc) == doc) { return scorer.explain(context.docBase); } @@ -375,6 +377,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); // nocommit do positions make sense here? + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int parentTarget) throws IOException { //System.out.println("Q.advance parentTarget=" + parentTarget); Index: lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java =================================================================== --- lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (.../trunk) (revision 1643331) +++ lucene/join/src/test/org/apache/lucene/search/join/TestBlockJoin.java (.../branches/lucene2878) (revision 1643331) @@ -25,14 +25,55 @@ import java.util.Locale; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.*; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.ReaderUtil; +import org.apache.lucene.index.StoredDocument; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.GroupDocs; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.*; +import org.apache.lucene.util.BitSet; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.TestUtil; public class TestBlockJoin extends LuceneTestCase { @@ -1148,7 +1189,7 @@ ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ScoreMode.Avg); Weight weight = s.createNormalizedWeight(q); - DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0), null); + DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0), DocsEnum.FLAG_FREQS, null); assertEquals(1, disi.advance(1)); r.close(); dir.close(); @@ -1182,7 +1223,7 @@ ToParentBlockJoinQuery q = new ToParentBlockJoinQuery(tq, parentFilter, ScoreMode.Avg); Weight weight = s.createNormalizedWeight(q); - DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0), null); + DocIdSetIterator disi = weight.scorer(s.getIndexReader().leaves().get(0), DocsEnum.FLAG_FREQS, null); assertEquals(2, disi.advance(0)); r.close(); dir.close(); Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java (.../trunk) (revision 1643331) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermVector.java (.../branches/lucene2878) (revision 1643331) @@ -16,6 +16,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + import java.io.IOException; import org.apache.lucene.analysis.TokenStream; @@ -23,7 +24,7 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; @@ -105,7 +106,7 @@ final TermsEnum termsEnum = vector.iterator(null); BytesRef termBytesRef; - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; //int sumFreq = 0; while ((termBytesRef = termsEnum.next()) != null) { //Grab the term (in same way as BytesRef.utf8ToString() but we don't want a String obj) Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java (.../trunk) (revision 0) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,55 @@ +package org.apache.lucene.search.highlight.positions; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.PositionsCollector; +import org.apache.lucene.search.posfilter.Interval; + +/** + * Collects the first maxDocs docs and their positions matching the query + * + * @lucene.experimental + */ + +public class HighlightingIntervalCollector extends PositionsCollector { + + int count; + DocAndPositions docs[]; + + public HighlightingIntervalCollector (int maxDocs) { + super(true); + docs = new DocAndPositions[maxDocs]; + } + + @Override + protected void collectPosition(int doc, Interval interval) { + if (count > docs.length) + return; // TODO can we indicate collection has finished somehow? + if (count <= 0 || docs[count - 1].doc != doc) { + DocAndPositions spdoc = new DocAndPositions(doc); + docs[count++] = spdoc; + } + docs[count - 1].storePosition(interval); + } + + public DocAndPositions[] getDocs () { + DocAndPositions ret[] = new DocAndPositions[count]; + System.arraycopy(docs, 0, ret, 0, count); + return ret; + } + +} Property changes on: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/HighlightingIntervalCollector.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsAndPositionsEnum.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsAndPositionsEnum.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/SingleDocsAndPositionsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,13 @@ * limitations under the License. */ -import org.apache.lucene.index.DocsAndPositionsEnum; +import java.io.IOException; + +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -class SingleDocsAndPositionsEnum extends DocsAndPositionsEnum { +class SingleDocsAndPositionsEnum extends DocsEnum { private int doc; private int pos; private int singleDocID; @@ -89,6 +91,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public BytesRef getPayload() { return payload; } Index: lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/codecs/idversion/IDVersionSegmentTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,7 @@ * limitations under the License. */ -import java.io.IOException; -import java.io.PrintStream; - import org.apache.lucene.codecs.BlockTermState; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.TermState; @@ -37,6 +33,9 @@ import org.apache.lucene.util.fst.PairOutputs.Pair; import org.apache.lucene.util.fst.Util; +import java.io.IOException; +import java.io.PrintStream; + /** Iterates through terms in this field; this class is public so users * can cast it to call {@link #seekExact(BytesRef, long)} for * optimistic-concurreny, and also {@link #getVersion} to get the @@ -1010,7 +1009,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits skipDocs, DocsEnum reuse, int flags) throws IOException { if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (.../trunk) (revision 1643331) +++ lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (.../branches/lucene2878) (revision 1643331) @@ -31,6 +31,7 @@ import org.apache.lucene.search.ComplexExplanation; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FakeScorer; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Query; @@ -133,7 +134,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - SVInnerScorer scorer = (SVInnerScorer) bulkScorer(context, false, null); + SVInnerScorer scorer = (SVInnerScorer) bulkScorer(context, false, DocsEnum.FLAG_FREQS, null); if (scorer != null) { return scorer.explain(doc); } @@ -163,7 +164,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { Terms terms = context.reader().terms(field); if (terms == null) { return null; @@ -181,10 +182,10 @@ } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { if (scoreDocsInOrder) { - return super.bulkScorer(context, scoreDocsInOrder, acceptDocs); + return super.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); } else { Terms terms = context.reader().terms(field); if (terms == null) { @@ -288,6 +289,7 @@ return new ComplexExplanation(true, scores[ords[scoreUpto]], "Score based on join value " + termsEnum.term().utf8ToString()); } + } // This impl that tracks whether a docid has already been emitted. This check makes sure that docs aren't emitted @@ -361,6 +363,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int docID() { return currentDoc; } Index: lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java (.../trunk) (revision 1643331) +++ lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java (.../branches/lucene2878) (revision 1643331) @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -34,6 +33,7 @@ import org.apache.lucene.util.BitDocIdSet; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; /** * Just like {@link ToParentBlockJoinQuery}, except this @@ -127,9 +127,9 @@ // NOTE: acceptDocs applies (and is checked) only in the // child document space @Override - public Scorer scorer(LeafReaderContext readerContext, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext readerContext, int flags, Bits acceptDocs) throws IOException { - final Scorer parentScorer = parentWeight.scorer(readerContext, null); + final Scorer parentScorer = parentWeight.scorer(readerContext, flags, null); if (parentScorer == null) { // No matches @@ -278,6 +278,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); // nocommit do positions make sense here? + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int childTarget) throws IOException { assert childTarget >= parentBits.length() || !parentBits.get(childTarget); Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/DocAndPositions.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/DocAndPositions.java (.../trunk) (revision 0) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/DocAndPositions.java (.../branches/lucene2878) (revision 1643331) @@ -0,0 +1,66 @@ +package org.apache.lucene.search.highlight.positions; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.posfilter.Interval; +import org.apache.lucene.util.ArrayUtil; + +import java.util.Comparator; + +/** Used to accumulate position intervals while scoring + * @lucene.experimental + */ +public final class DocAndPositions extends ScoreDoc { + + public int posCount = 0; + public Interval[] positions; + + public DocAndPositions(int doc) { + super(doc, 0); + positions = new Interval[32]; + } + + public void storePosition (Interval pos) { + ensureStorage(); + positions[posCount++] = (Interval) pos.clone(); + } + + private void ensureStorage () { + if (posCount >= positions.length) { + Interval temp[] = new Interval[positions.length * 2]; + System.arraycopy(positions, 0, temp, 0, positions.length); + positions = temp; + } + } + + public Interval[] sortedPositions() { + ArrayUtil.timSort(positions, 0, posCount, new Comparator() { + public int compare(Interval o1, Interval o2) { + return + o1.begin < o2.begin ? -1 : + (o1.begin > o2.begin ? 1 : + (o1.end < o2.end ? -1 : + (o1.end > o2.end ? 1 : + 0))); + } + + }); + return positions; + } +} Property changes on: lucene/highlighter/src/java/org/apache/lucene/search/highlight/positions/DocAndPositions.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java =================================================================== --- lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java (.../trunk) (revision 1643331) +++ lucene/sandbox/src/java/org/apache/lucene/search/TermAutomatonScorer.java (.../branches/lucene2878) (revision 1643331) @@ -326,6 +326,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; // nocommit we should be able to implement this + } + + @Override + public int startPosition() throws IOException { + return 0; // nocommit + } + + @Override + public int endPosition() throws IOException { + return 0; // nocommit + } + + @Override + public int startOffset() throws IOException { + return 0; // nocommit + } + + @Override + public int endOffset() throws IOException { + return 0; // nocommit + } + + @Override + public BytesRef getPayload() throws IOException { + return null; // nocommit + } + + @Override public int docID() { return docID; } Index: lucene/highlighter =================================================================== --- lucene/highlighter (.../trunk) (revision 1643331) +++ lucene/highlighter (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/highlighter ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/highlighter:r1638800-1643324 Index: lucene/join =================================================================== --- lucene/join (.../trunk) (revision 1643331) +++ lucene/join (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/join ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/join:r1638800-1642714 Index: lucene/sandbox =================================================================== --- lucene/sandbox (.../trunk) (revision 1643331) +++ lucene/sandbox (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/sandbox ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/sandbox:r1638800-1642714 Index: lucene/queryparser =================================================================== --- lucene/queryparser (.../trunk) (revision 1643331) +++ lucene/queryparser (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/queryparser ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/queryparser:r1638800-1642714 Index: lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java =================================================================== --- lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java (.../trunk) (revision 1643331) +++ lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndexAgainstRAMDir.java (.../branches/lucene2878) (revision 1643331) @@ -42,7 +42,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.CompositeReader; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexOptions; @@ -68,8 +67,8 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.ByteBlockPool.Allocator; -import org.apache.lucene.util.ByteBlockPool; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.LineFileDocs; @@ -198,9 +197,9 @@ while(iwTermsIter.next() != null) { assertNotNull(memTermsIter.next()); assertEquals(iwTermsIter.term(), memTermsIter.term()); - DocsAndPositionsEnum iwDocsAndPos = iwTermsIter.docsAndPositions(null, null); - DocsAndPositionsEnum memDocsAndPos = memTermsIter.docsAndPositions(null, null); - while(iwDocsAndPos.nextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { + DocsEnum iwDocsAndPos = iwTermsIter.docsAndPositions(null, null); + DocsEnum memDocsAndPos = memTermsIter.docsAndPositions(null, null); + while(iwDocsAndPos.nextDoc() != DocsEnum.NO_MORE_DOCS) { assertEquals(iwDocsAndPos.docID(), memDocsAndPos.nextDoc()); assertEquals(iwDocsAndPos.freq(), memDocsAndPos.freq()); for (int i = 0; i < iwDocsAndPos.freq(); i++) { @@ -219,7 +218,7 @@ assertEquals(iwTermsIter.term(), memTermsIter.term()); DocsEnum iwDocsAndPos = iwTermsIter.docs(null, null); DocsEnum memDocsAndPos = memTermsIter.docs(null, null); - while(iwDocsAndPos.nextDoc() != DocsAndPositionsEnum.NO_MORE_DOCS) { + while(iwDocsAndPos.nextDoc() != DocsEnum.NO_MORE_DOCS) { assertEquals(iwDocsAndPos.docID(), memDocsAndPos.nextDoc()); assertEquals(iwDocsAndPos.freq(), memDocsAndPos.freq()); } @@ -345,7 +344,7 @@ memory.addField("foo", "bar", analyzer); LeafReader reader = (LeafReader) memory.createSearcher().getIndexReader(); assertEquals(1, reader.terms("foo").getSumTotalTermFreq()); - DocsAndPositionsEnum disi = reader.termPositionsEnum(new Term("foo", "bar")); + DocsEnum disi = reader.termPositionsEnum(new Term("foo", "bar")); int docid = disi.docID(); assertEquals(-1, docid); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -517,8 +516,8 @@ assertNotNull(memTermEnum.next()); assertThat(termEnum.totalTermFreq(), equalTo(memTermEnum.totalTermFreq())); - DocsAndPositionsEnum docsPosEnum = termEnum.docsAndPositions(null, null, 0); - DocsAndPositionsEnum memDocsPosEnum = memTermEnum.docsAndPositions(null, null, 0); + DocsEnum docsPosEnum = termEnum.docsAndPositions(null, null, 0); + DocsEnum memDocsPosEnum = memTermEnum.docsAndPositions(null, null, 0); String currentTerm = termEnum.term().utf8ToString(); assertThat("Token mismatch for field: " + field_name, currentTerm, equalTo(memTermEnum.term().utf8ToString())); Index: lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,14 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; -import java.io.IOException; - /** * {@link Scorer} which returns the result of {@link FunctionValues#floatVal(int)} as * the score for a document. @@ -93,6 +94,36 @@ } @Override + public int nextPosition() throws IOException { + return NO_MORE_POSITIONS; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public long cost() { return maxDoc; } Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/QueryValueSource.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,16 @@ package org.apache.lucene.queries.function.valuesource; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.FloatDocValues; -import org.apache.lucene.search.*; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.apache.lucene.util.mutable.MutableValue; import org.apache.lucene.util.mutable.MutableValueFloat; @@ -123,7 +127,7 @@ try { if (doc < lastDocRequested) { if (noMatches) return defVal; - scorer = weight.scorer(readerContext, acceptDocs); + scorer = weight.scorer(readerContext, DocsEnum.FLAG_FREQS, acceptDocs); if (scorer==null) { noMatches = true; return defVal; @@ -154,7 +158,7 @@ try { if (doc < lastDocRequested) { if (noMatches) return false; - scorer = weight.scorer(readerContext, acceptDocs); + scorer = weight.scorer(readerContext, DocsEnum.FLAG_FREQS, acceptDocs); scorerDoc = -1; if (scorer==null) { noMatches = true; @@ -212,7 +216,7 @@ mval.exists = false; return; } - scorer = weight.scorer(readerContext, acceptDocs); + scorer = weight.scorer(readerContext, DocsEnum.FLAG_FREQS, acceptDocs); scorerDoc = -1; if (scorer==null) { noMatches = true; Index: lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (.../branches/lucene2878) (revision 1643331) @@ -18,23 +18,24 @@ */ import java.io.IOException; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Set; -import java.util.Arrays; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.FunctionQuery; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.ComplexExplanation; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ToStringUtils; /** @@ -234,14 +235,14 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { - Scorer subQueryScorer = subQueryWeight.scorer(context, acceptDocs); + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + Scorer subQueryScorer = subQueryWeight.scorer(context, flags, acceptDocs); if (subQueryScorer == null) { return null; } Scorer[] valSrcScorers = new Scorer[valSrcWeights.length]; for(int i = 0; i < valSrcScorers.length; i++) { - valSrcScorers[i] = valSrcWeights[i].scorer(context, acceptDocs); + valSrcScorers[i] = valSrcWeights[i].scorer(context, flags, acceptDocs); } return new CustomScorer(CustomScoreQuery.this.getCustomScoreProvider(context), this, queryWeight, subQueryScorer, valSrcScorers); } @@ -291,6 +292,8 @@ private final CustomScoreProvider provider; private final float[] vScores; // reused in score() to avoid allocating this array for each doc + // TODO : can we use FilterScorer here instead? + // constructor private CustomScorer(CustomScoreProvider provider, CustomWeight w, float qWeight, Scorer subQueryScorer, Scorer[] valSrcScorers) { @@ -333,6 +336,36 @@ } @Override + public int nextPosition() throws IOException { + return subQueryScorer.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + return subQueryScorer.startPosition(); + } + + @Override + public int endPosition() throws IOException { + return subQueryScorer.endPosition(); + } + + @Override + public int startOffset() throws IOException { + return subQueryScorer.startOffset(); + } + + @Override + public int endOffset() throws IOException { + return subQueryScorer.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + return subQueryScorer.getPayload(); + } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(subQueryScorer, "CUSTOM")); } Index: lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (.../trunk) (revision 1643331) +++ lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (.../branches/lucene2878) (revision 1643331) @@ -33,7 +33,6 @@ import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -968,19 +967,16 @@ @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) { - if (reuse == null || !(reuse instanceof MemoryDocsEnum)) { - reuse = new MemoryDocsEnum(); - } - return ((MemoryDocsEnum) reuse).reset(liveDocs, info.sliceArray.freq[info.sortedTerms[termUpto]]); + return docsAndPositions(liveDocs, reuse, flags); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { - if (reuse == null || !(reuse instanceof MemoryDocsAndPositionsEnum)) { - reuse = new MemoryDocsAndPositionsEnum(); + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) { + if (reuse == null || !(reuse instanceof MemoryDocsEnum)) { + reuse = new MemoryDocsEnum(); } final int ord = info.sortedTerms[termUpto]; - return ((MemoryDocsAndPositionsEnum) reuse).reset(liveDocs, info.sliceArray.start[ord], info.sliceArray.end[ord], info.sliceArray.freq[ord]); + return ((MemoryDocsEnum) reuse).reset(liveDocs, info.sliceArray.start[ord], info.sliceArray.end[ord], info.sliceArray.freq[ord]); } @Override @@ -998,51 +994,6 @@ } private class MemoryDocsEnum extends DocsEnum { - private boolean hasNext; - private Bits liveDocs; - private int doc = -1; - private int freq; - - public DocsEnum reset(Bits liveDocs, int freq) { - this.liveDocs = liveDocs; - hasNext = true; - doc = -1; - this.freq = freq; - return this; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() { - if (hasNext && (liveDocs == null || liveDocs.get(0))) { - hasNext = false; - return doc = 0; - } else { - return doc = NO_MORE_DOCS; - } - } - - @Override - public int advance(int target) throws IOException { - return slowAdvance(target); - } - - @Override - public int freq() throws IOException { - return freq; - } - - @Override - public long cost() { - return 1; - } - } - - private class MemoryDocsAndPositionsEnum extends DocsAndPositionsEnum { private int posUpto; // for assert private boolean hasNext; private Bits liveDocs; @@ -1049,14 +1000,15 @@ private int doc = -1; private SliceReader sliceReader; private int freq; + private int pos; private int startOffset; private int endOffset; - public MemoryDocsAndPositionsEnum() { + public MemoryDocsEnum() { this.sliceReader = new SliceReader(intBlockPool); } - public DocsAndPositionsEnum reset(Bits liveDocs, int start, int end, int freq) { + public DocsEnum reset(Bits liveDocs, int start, int end, int freq) { this.liveDocs = liveDocs; this.sliceReader.reset(start, end); posUpto = 0; // for assert @@ -1074,6 +1026,7 @@ @Override public int nextDoc() { + pos = -1; if (hasNext && (liveDocs == null || liveDocs.get(0))) { hasNext = false; return doc = 0; @@ -1094,10 +1047,12 @@ @Override public int nextPosition() { - assert posUpto++ < freq; + //assert posUpto++ < freq; + if (posUpto++ >= freq) + return NO_MORE_POSITIONS; assert !sliceReader.endOfSlice() : " stores offsets : " + startOffset; if (storeOffsets) { - int pos = sliceReader.readInt(); + pos = sliceReader.readInt(); startOffset = sliceReader.readInt(); endOffset = sliceReader.readInt(); return pos; @@ -1107,6 +1062,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() { return startOffset; } Index: lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,17 +17,24 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; -import java.io.IOException; -import java.util.Set; -import java.util.Map; - /** * Returns a score for each document based on a ValueSource, * often some function of the value of a field. @@ -89,13 +96,13 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new AllScorer(context, acceptDocs, this, queryWeight); } @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - return ((AllScorer)scorer(context, context.reader().getLiveDocs())).explain(doc); + return ((AllScorer)scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs())).explain(doc); } } @@ -166,6 +173,36 @@ return 1; } + @Override + public int nextPosition() throws IOException { + return NO_MORE_POSITIONS; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + public Explanation explain(int doc) throws IOException { float sc = qWeight * vals.floatVal(doc); @@ -177,6 +214,7 @@ result.addDetail(new Explanation(weight.queryNorm,"queryNorm")); return result; } + } Index: lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java (.../branches/lucene2878) (revision 1643331) @@ -17,19 +17,25 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.*; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FilterScorer; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; -import java.util.Map; - /** * Query that is boosted by a ValueSource */ @@ -97,8 +103,8 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { - Scorer subQueryScorer = qWeight.scorer(context, acceptDocs); + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + Scorer subQueryScorer = qWeight.scorer(context, flags, acceptDocs); if (subQueryScorer == null) { return null; } @@ -122,41 +128,24 @@ } - private class CustomScorer extends Scorer { + private class CustomScorer extends FilterScorer { private final BoostedQuery.BoostedWeight weight; private final float qWeight; - private final Scorer scorer; private final FunctionValues vals; private final LeafReaderContext readerContext; private CustomScorer(LeafReaderContext readerContext, BoostedQuery.BoostedWeight w, float qWeight, Scorer scorer, ValueSource vs) throws IOException { - super(w); + super(scorer); this.weight = w; this.qWeight = qWeight; - this.scorer = scorer; this.readerContext = readerContext; this.vals = vs.getValues(weight.fcontext, readerContext); } - @Override - public int docID() { - return scorer.docID(); - } - - @Override - public int advance(int target) throws IOException { - return scorer.advance(target); - } - - @Override - public int nextDoc() throws IOException { - return scorer.nextDoc(); - } - @Override public float score() throws IOException { - float score = qWeight * scorer.score() * vals.floatVal(scorer.docID()); + float score = qWeight * in.score() * vals.floatVal(in.docID()); // Current Lucene priority queues can't handle NaN and -Infinity, so // map to -Float.MAX_VALUE. This conditional handles both -infinity @@ -165,13 +154,8 @@ } @Override - public int freq() throws IOException { - return scorer.freq(); - } - - @Override public Collection getChildren() { - return Collections.singleton(new ChildScorer(scorer, "CUSTOM")); + return Collections.singleton(new ChildScorer(in, "CUSTOM")); } public Explanation explain(int doc) throws IOException { @@ -187,10 +171,6 @@ return res; } - @Override - public long cost() { - return scorer.cost(); - } } Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,19 @@ package org.apache.lucene.queries.function.valuesource; -import org.apache.lucene.index.*; +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.IntDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; -import java.io.IOException; -import java.util.Map; - /** * Function that returns {@link DocsEnum#freq()} for the * supplied term in every document. @@ -77,6 +81,36 @@ } @Override + public int nextPosition() throws IOException { + return NO_MORE_POSITIONS; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int docID() { return DocIdSetIterator.NO_MORE_DOCS; } Index: lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java (.../trunk) (revision 1643331) +++ lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java (.../branches/lucene2878) (revision 1643331) @@ -17,7 +17,14 @@ * limitations under the License. */ -import org.apache.lucene.index.*; +import java.io.IOException; +import java.util.Map; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.DocIdSetIterator; @@ -25,9 +32,6 @@ import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.util.BytesRef; -import java.io.IOException; -import java.util.Map; - /** * Function that returns {@link TFIDFSimilarity#tf(float)} * for every document. @@ -84,6 +88,36 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int docID() { return DocIdSetIterator.NO_MORE_DOCS; } Index: lucene/queries =================================================================== --- lucene/queries (.../trunk) (revision 1643331) +++ lucene/queries (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/queries ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/queries:r1638800-1643324 Index: lucene/memory =================================================================== --- lucene/memory (.../trunk) (revision 1643331) +++ lucene/memory (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/memory ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/memory:r1638800-1643324 Index: lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionRescorer.java =================================================================== --- lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionRescorer.java (.../trunk) (revision 1643331) +++ lucene/expressions/src/java/org/apache/lucene/expressions/ExpressionRescorer.java (.../branches/lucene2878) (revision 1643331) @@ -18,7 +18,6 @@ */ import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -27,12 +26,11 @@ import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.FakeScorer; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Rescorer; -import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortRescorer; -import org.apache.lucene.search.Weight; /** * A {@link Rescorer} that uses an expression to re-score @@ -58,56 +56,6 @@ this.bindings = bindings; } - private static class FakeScorer extends Scorer { - float score; - int doc = -1; - int freq = 1; - - public FakeScorer() { - super(null); - } - - @Override - public int advance(int target) { - throw new UnsupportedOperationException("FakeScorer doesn't support advance(int)"); - } - - @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - return freq; - } - - @Override - public int nextDoc() { - throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()"); - } - - @Override - public float score() { - return score; - } - - @Override - public long cost() { - return 1; - } - - @Override - public Weight getWeight() { - throw new UnsupportedOperationException(); - } - - @Override - public Collection getChildren() { - throw new UnsupportedOperationException(); - } - } - @Override public Explanation explain(IndexSearcher searcher, Explanation firstPassExplanation, int docID) throws IOException { Explanation result = super.explain(searcher, firstPassExplanation, docID); Index: lucene/expressions =================================================================== --- lucene/expressions (.../trunk) (revision 1643331) +++ lucene/expressions (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/expressions ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/expressions:r1638800-1642144 Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/TaxonomyIndexArrays.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/TaxonomyIndexArrays.java (.../trunk) (revision 1643331) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/directory/TaxonomyIndexArrays.java (.../branches/lucene2878) (revision 1643331) @@ -1,16 +1,16 @@ package org.apache.lucene.facet.taxonomy.directory; -import java.io.IOException; - import org.apache.lucene.facet.taxonomy.ParallelTaxonomyArrays; import org.apache.lucene.facet.taxonomy.TaxonomyReader; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.MultiFields; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.ArrayUtil; +import java.io.IOException; + /* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -129,9 +129,9 @@ // it's ok to use MultiFields because we only iterate on one posting list. // breaking it to loop over the leaves() only complicates code for no // apparent gain. - DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader, null, + DocsEnum positions = MultiFields.getTermPositionsEnum(reader, null, Consts.FIELD_PAYLOADS, Consts.PAYLOAD_PARENT_BYTES_REF, - DocsAndPositionsEnum.FLAG_PAYLOADS); + DocsEnum.FLAG_PAYLOADS); // shouldn't really happen, if it does, something's wrong if (positions == null || positions.advance(first) == DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java (.../trunk) (revision 1643331) +++ lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysQuery.java (.../branches/lucene2878) (revision 1643331) @@ -16,11 +16,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; -import java.util.Arrays; - +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; @@ -29,10 +27,11 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; +import java.io.IOException; +import java.util.Arrays; /** Only purpose is to punch through and return a * DrillSidewaysScorer */ @@ -118,17 +117,17 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { // We can only run as a top scorer: throw new UnsupportedOperationException(); } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { // TODO: it could be better if we take acceptDocs // into account instead of baseScorer? - Scorer baseScorer = baseWeight.scorer(context, acceptDocs); + Scorer baseScorer = baseWeight.scorer(context, flags, acceptDocs); DrillSidewaysScorer.DocsAndCost[] dims = new DrillSidewaysScorer.DocsAndCost[drillDowns.length]; int nullCount = 0; @@ -173,7 +172,7 @@ dims[dim].disi = disi; } } else { - DocIdSetIterator disi = ((Weight) drillDowns[dim]).scorer(context, null); + DocIdSetIterator disi = ((Weight) drillDowns[dim]).scorer(context, flags, null); if (disi == null) { nullCount++; continue; Index: lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java (.../trunk) (revision 1643331) +++ lucene/facet/src/java/org/apache/lucene/facet/DrillSidewaysScorer.java (.../branches/lucene2878) (revision 1643331) @@ -21,15 +21,16 @@ import java.util.Collection; import java.util.Collections; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.BulkScorer; import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.FixedBitSet; class DrillSidewaysScorer extends BulkScorer { @@ -645,6 +646,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException("FakeScorer doesn't support nextPosition()"); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int nextDoc() { throw new UnsupportedOperationException("FakeScorer doesn't support nextDoc()"); } Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java (.../trunk) (revision 1643331) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumValueSource.java (.../branches/lucene2878) (revision 1643331) @@ -18,13 +18,12 @@ */ import java.io.IOException; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.facet.FacetsCollector; import org.apache.lucene.facet.FacetsCollector.MatchingDocs; -import org.apache.lucene.facet.FacetsCollector; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.FunctionValues; @@ -31,8 +30,8 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FakeScorer; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Weight; import org.apache.lucene.util.IntsRef; /** Aggregates sum of values from {@link @@ -62,28 +61,6 @@ sumValues(fc.getMatchingDocs(), fc.getKeepScores(), valueSource); } - private static final class FakeScorer extends Scorer { - float score; - int docID; - FakeScorer() { super(null); } - @Override public float score() throws IOException { return score; } - @Override public int freq() throws IOException { throw new UnsupportedOperationException(); } - @Override public int docID() { return docID; } - @Override public int nextDoc() throws IOException { throw new UnsupportedOperationException(); } - @Override public int advance(int target) throws IOException { throw new UnsupportedOperationException(); } - @Override public long cost() { return 0; } - - @Override - public Weight getWeight() { - throw new UnsupportedOperationException(); - } - - @Override - public Collection getChildren() { - throw new UnsupportedOperationException(); - } - } - private final void sumValues(List matchingDocs, boolean keepScores, ValueSource valueSource) throws IOException { final FakeScorer scorer = new FakeScorer(); Map context = new HashMap<>(); @@ -104,7 +81,7 @@ while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { ords.get(doc, scratch); if (keepScores) { - scorer.docID = doc; + scorer.doc = doc; scorer.score = scores[scoresIdx++]; } float value = (float) functionValues.doubleVal(doc); Index: lucene/facet =================================================================== --- lucene/facet (.../trunk) (revision 1643331) +++ lucene/facet (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/facet ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/facet:r1638800-1643324 Index: lucene/benchmark =================================================================== --- lucene/benchmark (.../trunk) (revision 1643331) +++ lucene/benchmark (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/benchmark ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/benchmark:r1638800-1642714 Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (.../trunk) (revision 1643331) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (.../branches/lucene2878) (revision 1643331) @@ -31,7 +31,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Terms; @@ -111,7 +111,7 @@ TermsEnum termsEnum = vector.iterator(null); termsEnum.next(); assertEquals(2, termsEnum.totalTermFreq()); - DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null); + DocsEnum positions = termsEnum.docsAndPositions(null, null); assertTrue(positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(2, positions.freq()); positions.nextPosition(); Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java (.../trunk) (revision 1643331) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestClassicAnalyzer.java (.../branches/lucene2878) (revision 1643331) @@ -7,7 +7,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -281,7 +281,7 @@ // Make sure position is still incremented when // massive term is skipped: - DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, + DocsEnum tps = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "content", new BytesRef("another")); Index: lucene/analysis/common =================================================================== --- lucene/analysis/common (.../trunk) (revision 1643331) +++ lucene/analysis/common (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/analysis/common ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/analysis/common:r1638800-1642714 Index: lucene/analysis =================================================================== --- lucene/analysis (.../trunk) (revision 1643331) +++ lucene/analysis (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/analysis ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/analysis:r1638800-1642714 Index: lucene/build.xml =================================================================== --- lucene/build.xml (.../trunk) (revision 1643331) +++ lucene/build.xml (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/build.xml:r1638800-1642714 Index: lucene/tools =================================================================== --- lucene/tools (.../trunk) (revision 1643331) +++ lucene/tools (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/tools ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/tools:r1638800-1642144 Index: lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java =================================================================== --- lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (.../trunk) (revision 1643331) +++ lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (.../branches/lucene2878) (revision 1643331) @@ -18,11 +18,29 @@ package org.apache.lucene.search.grouping; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import com.carrotsearch.randomizedtesting.annotations.Seed; import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; @@ -34,7 +52,20 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.CachingCollector; +import org.apache.lucene.search.CachingWrapperFilter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MultiCollector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryWrapperFilter; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.function.FunctionAllGroupsCollector; import org.apache.lucene.search.grouping.function.FunctionFirstPassGroupingCollector; import org.apache.lucene.search.grouping.function.FunctionSecondPassGroupingCollector; @@ -54,6 +85,7 @@ // - test ties // - test compound sort +@Seed("3C4E441C6A8DA6A2:4E026113DBED10D1") public class TestGrouping extends LuceneTestCase { public void testBasic() throws Exception { @@ -878,7 +910,7 @@ } } else { // Collect only into cache, then replay multiple times: - c = cCache = CachingCollector.create(false, true, maxCacheMB); + c = cCache = CachingCollector.create(false, DocsEnum.FLAG_FREQS, true, maxCacheMB); } } else { cCache = null; Index: lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java (.../trunk) (revision 1643331) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/SortingLeafReader.java (.../branches/lucene2878) (revision 1643331) @@ -23,7 +23,6 @@ import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.FieldInfos; @@ -52,7 +51,7 @@ /** * An {@link org.apache.lucene.index.LeafReader} which supports sorting documents by a given * {@link Sort}. You can use this class to sort an index as follows: - * + * * 

      * IndexWriter writer; // writer to which the sorted index will be added
  * DirectoryReader reader; // reader on the input index
@@ -62,7 +61,7 @@
  * writer.close();
  * reader.close();
  *
    - * + * * @lucene.experimental */ public class SortingLeafReader extends FilterLeafReader { @@ -94,7 +93,7 @@ private final Sorter.DocMap docMap; private final IndexOptions indexOptions; - + public SortingTerms(final Terms in, IndexOptions indexOptions, final Sorter.DocMap docMap) { super(in); this.docMap = docMap; @@ -118,7 +117,7 @@ final Sorter.DocMap docMap; // pkg-protected to avoid synthetic accessor methods private final IndexOptions indexOptions; - + public SortingTermsEnum(final TermsEnum in, Sorter.DocMap docMap, IndexOptions indexOptions) { super(in); this.docMap = docMap; @@ -146,6 +145,10 @@ @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, final int flags) throws IOException { + + if ((flags & DocsEnum.FLAG_POSITIONS) >= DocsEnum.FLAG_POSITIONS) + return docsAndPositions(liveDocs, reuse, flags); + final DocsEnum inReuse; final SortingDocsEnum wrapReuse; if (reuse != null && reuse instanceof SortingDocsEnum) { @@ -164,8 +167,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, final int flags) throws IOException { - final DocsAndPositionsEnum inReuse; + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, final int flags) throws IOException { + final DocsEnum inReuse; final SortingDocsAndPositionsEnum wrapReuse; if (reuse != null && reuse instanceof SortingDocsAndPositionsEnum) { // if we're asked to reuse the given DocsEnum and it is Sorting, return @@ -177,7 +180,7 @@ inReuse = reuse; } - final DocsAndPositionsEnum inDocsAndPositions = in.docsAndPositions(newToOld(liveDocs), inReuse, flags); + final DocsEnum inDocsAndPositions = in.docsAndPositions(newToOld(liveDocs), inReuse, flags); if (inDocsAndPositions == null) { return null; } @@ -193,10 +196,10 @@ } private static class SortingBinaryDocValues extends BinaryDocValues { - + private final BinaryDocValues in; private final Sorter.DocMap docMap; - + SortingBinaryDocValues(BinaryDocValues in, Sorter.DocMap docMap) { this.in = in; this.docMap = docMap; @@ -207,7 +210,7 @@ return in.get(docMap.newToOld(docID)); } } - + private static class SortingNumericDocValues extends NumericDocValues { private final NumericDocValues in; @@ -223,33 +226,33 @@ return in.get(docMap.newToOld(docID)); } } - + private static class SortingSortedNumericDocValues extends SortedNumericDocValues { - + private final SortedNumericDocValues in; private final Sorter.DocMap docMap; - + SortingSortedNumericDocValues(SortedNumericDocValues in, DocMap docMap) { this.in = in; this.docMap = docMap; } - + @Override public int count() { return in.count(); } - + @Override public void setDocument(int doc) { in.setDocument(docMap.newToOld(doc)); } - + @Override public long valueAt(int index) { return in.valueAt(index); } } - + private static class SortingBits implements Bits { private final Bits in; @@ -270,12 +273,12 @@ return in.length(); } } - + private static class SortingSortedDocValues extends SortedDocValues { - + private final SortedDocValues in; private final Sorter.DocMap docMap; - + SortingSortedDocValues(SortedDocValues in, Sorter.DocMap docMap) { this.in = in; this.docMap = docMap; @@ -306,12 +309,12 @@ return in.lookupTerm(key); } } - + private static class SortingSortedSetDocValues extends SortedSetDocValues { - + private final SortedSetDocValues in; private final Sorter.DocMap docMap; - + SortingSortedSetDocValues(SortedSetDocValues in, Sorter.DocMap docMap) { this.in = in; this.docMap = docMap; @@ -344,14 +347,14 @@ } static class SortingDocsEnum extends FilterDocsEnum { - + private static final class DocFreqSorter extends TimSorter { - + private int[] docs; private int[] freqs; private final int[] tmpDocs; private int[] tmpFreqs; - + public DocFreqSorter(int maxDoc) { super(maxDoc / 64); this.tmpDocs = new int[maxDoc / 64]; @@ -369,13 +372,13 @@ protected int compare(int i, int j) { return docs[i] - docs[j]; } - + @Override protected void swap(int i, int j) { int tmpDoc = docs[i]; docs[i] = docs[j]; docs[j] = tmpDoc; - + if (freqs != null) { int tmpFreq = freqs[i]; freqs[i] = freqs[j]; @@ -483,31 +486,31 @@ // don't bother to implement efficiently for now. return slowAdvance(target); } - + @Override public int docID() { return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt]; } - + @Override public int freq() throws IOException { return withFreqs && docIt < upto ? freqs[docIt] : 1; } - + @Override public int nextDoc() throws IOException { if (++docIt >= upto) return NO_MORE_DOCS; return docs[docIt]; } - + /** Returns the wrapped {@link DocsEnum}. */ DocsEnum getWrapped() { return in; } } - - static class SortingDocsAndPositionsEnum extends FilterDocsAndPositionsEnum { - + + static class SortingDocsAndPositionsEnum extends FilterDocsEnum { + /** * A {@link TimSorter} which sorts two parallel arrays of doc IDs and * offsets in one go. Everytime a doc ID is 'swapped', its correponding offset @@ -514,12 +517,12 @@ * is swapped too. */ private static final class DocOffsetSorter extends TimSorter { - + private int[] docs; private long[] offsets; private final int[] tmpDocs; private final long[] tmpOffsets; - + public DocOffsetSorter(int maxDoc) { super(maxDoc / 64); this.tmpDocs = new int[maxDoc / 64]; @@ -535,13 +538,13 @@ protected int compare(int i, int j) { return docs[i] - docs[j]; } - + @Override protected void swap(int i, int j) { int tmpDoc = docs[i]; docs[i] = docs[j]; docs[j] = tmpDoc; - + long tmpOffset = offsets[i]; offsets[i] = offsets[j]; offsets[j] = tmpOffset; @@ -570,16 +573,16 @@ return tmpDocs[i] - docs[j]; } } - + private final int maxDoc; private final DocOffsetSorter sorter; private int[] docs; private long[] offsets; private final int upto; - + private final IndexInput postingInput; private final boolean storeOffsets; - + private int docIt = -1; private int pos; private int startOffset = -1; @@ -589,7 +592,7 @@ private final RAMFile file; - SortingDocsAndPositionsEnum(int maxDoc, SortingDocsAndPositionsEnum reuse, final DocsAndPositionsEnum in, Sorter.DocMap docMap, boolean storeOffsets) throws IOException { + SortingDocsAndPositionsEnum(int maxDoc, SortingDocsAndPositionsEnum reuse, final DocsEnum in, Sorter.DocMap docMap, boolean storeOffsets) throws IOException { super(in); this.maxDoc = maxDoc; this.storeOffsets = storeOffsets; @@ -632,7 +635,7 @@ } // for testing - boolean reused(DocsAndPositionsEnum other) { + boolean reused(DocsEnum other) { if (other == null || !(other instanceof SortingDocsAndPositionsEnum)) { return false; } @@ -639,7 +642,7 @@ return docs == ((SortingDocsAndPositionsEnum) other).docs; } - private void addPositions(final DocsAndPositionsEnum in, final IndexOutput out) throws IOException { + private void addPositions(final DocsEnum in, final IndexOutput out) throws IOException { int freq = in.freq(); out.writeVInt(freq); int previousPosition = 0; @@ -648,7 +651,7 @@ final int pos = in.nextPosition(); final BytesRef payload = in.getPayload(); // The low-order bit of token is set only if there is a payload, the - // previous bits are the delta-encoded position. + // previous bits are the delta-encoded position. final int token = (pos - previousPosition) << 1 | (payload == null ? 0 : 1); out.writeVInt(token); previousPosition = pos; @@ -665,7 +668,7 @@ } } } - + @Override public int advance(final int target) throws IOException { // need to support it for checkIndex, but in practice it won't be called, so @@ -672,27 +675,27 @@ // don't bother to implement efficiently for now. return slowAdvance(target); } - + @Override public int docID() { return docIt < 0 ? -1 : docIt >= upto ? NO_MORE_DOCS : docs[docIt]; } - + @Override public int endOffset() throws IOException { return endOffset; } - + @Override public int freq() throws IOException { return currFreq; } - + @Override public BytesRef getPayload() throws IOException { return payload.length == 0 ? null : payload; } - + @Override public int nextDoc() throws IOException { if (++docIt >= upto) return DocIdSetIterator.NO_MORE_DOCS; @@ -703,7 +706,7 @@ endOffset = 0; return docs[docIt]; } - + @Override public int nextPosition() throws IOException { final int token = postingInput.readVInt(); @@ -724,14 +727,14 @@ } return pos; } - + @Override public int startOffset() throws IOException { return startOffset; } - /** Returns the wrapped {@link DocsAndPositionsEnum}. */ - DocsAndPositionsEnum getWrapped() { + /** Returns the wrapped {@link DocsEnum}. */ + DocsEnum getWrapped() { return in; } } @@ -767,12 +770,12 @@ public void document(final int docID, final StoredFieldVisitor visitor) throws IOException { in.document(docMap.newToOld(docID), visitor); } - + @Override public Fields fields() throws IOException { return new SortingFields(in.fields(), in.getFieldInfos(), docMap); } - + @Override public BinaryDocValues getBinaryDocValues(String field) throws IOException { BinaryDocValues oldDocValues = in.getBinaryDocValues(field); @@ -782,7 +785,7 @@ return new SortingBinaryDocValues(oldDocValues, docMap); } } - + @Override public Bits getLiveDocs() { final Bits inLiveDocs = in.getLiveDocs(); @@ -792,7 +795,7 @@ return new SortingBits(inLiveDocs, docMap); } } - + @Override public NumericDocValues getNormValues(String field) throws IOException { final NumericDocValues norm = in.getNormValues(field); @@ -809,7 +812,7 @@ if (oldDocValues == null) return null; return new SortingNumericDocValues(oldDocValues, docMap); } - + @Override public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { @@ -830,7 +833,7 @@ return new SortingSortedDocValues(sortedDV, docMap); } } - + @Override public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { SortedSetDocValues sortedSetDV = in.getSortedSetDocValues(field); @@ -838,7 +841,7 @@ return null; } else { return new SortingSortedSetDocValues(sortedSetDV, docMap); - } + } } @Override @@ -855,5 +858,5 @@ public Fields getTermVectors(final int docID) throws IOException { return in.getTermVectors(docMap.newToOld(docID)); } - + } Index: lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java =================================================================== --- lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java (.../trunk) (revision 1643331) +++ lucene/spatial/src/test/org/apache/lucene/spatial/prefix/tree/DateRangePrefixTreeTest.java (.../branches/lucene2878) (revision 1643331) @@ -157,9 +157,7 @@ assertEquals("2014", tree.parseShape("[2014-01-01 TO 2014-12-31]").toString()); - assertEquals("2014", tree.parseShape("[2014-01 TO 2014]").toString()); - assertEquals("2014-01", tree.parseShape("[2014 TO 2014-01]").toString()); - assertEquals("2014-12", tree.parseShape("[2014-12 TO 2014]").toString()); + assertEquals("2014", tree.parseShape("[2014-01 TO 2014]").toString()); assertEquals("[2014 TO 2014-04-06]", tree.parseShape("[2014-01 TO 2014-04-06]").toString()); Index: lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java =================================================================== --- lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (.../trunk) (revision 1643331) +++ lucene/misc/src/test/org/apache/lucene/index/sorter/SorterTestBase.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,6 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Random; - import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -31,8 +24,8 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; @@ -42,7 +35,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexOptions; @@ -57,9 +49,8 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.sorter.SortingLeafReader.SortingDocsAndPositionsEnum; import org.apache.lucene.index.sorter.SortingLeafReader.SortingDocsEnum; import org.apache.lucene.search.CollectionStatistics; import org.apache.lucene.search.DocIdSetIterator; @@ -75,6 +66,13 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; + public abstract class SorterTestBase extends LuceneTestCase { static final class NormsSimilarity extends Similarity { @@ -254,7 +252,7 @@ public void testDocsAndPositionsEnum() throws Exception { TermsEnum termsEnum = sortedReader.terms(DOC_POSITIONS_FIELD).iterator(null); assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(new BytesRef(DOC_POSITIONS_TERM))); - DocsAndPositionsEnum sortedPositions = termsEnum.docsAndPositions(null, null); + DocsEnum sortedPositions = termsEnum.docsAndPositions(null, null); int doc; // test nextDoc() @@ -270,10 +268,10 @@ } // test advance() - final DocsAndPositionsEnum reuse = sortedPositions; + final DocsEnum reuse = sortedPositions; sortedPositions = termsEnum.docsAndPositions(null, reuse); - if (sortedPositions instanceof SortingDocsAndPositionsEnum) { - assertTrue(((SortingDocsAndPositionsEnum) sortedPositions).reused(reuse)); // make sure reuse worked + if (sortedPositions instanceof SortingDocsEnum) { + assertTrue(((SortingDocsEnum) sortedPositions).reused(reuse)); // make sure reuse worked } doc = 0; while ((doc = sortedPositions.advance(doc + TestUtil.nextInt(random(), 1, 5))) != DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (.../trunk) (revision 1643331) +++ lucene/misc/src/java/org/apache/lucene/index/sorter/Sorter.java (.../branches/lucene2878) (revision 1643331) @@ -20,8 +20,9 @@ import java.io.IOException; import java.util.Comparator; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.FakeScorer; import org.apache.lucene.search.FieldComparator; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.Sort; @@ -261,25 +262,6 @@ return getID(); } - static final Scorer FAKESCORER = new Scorer(null) { - - @Override - public float score() throws IOException { throw new UnsupportedOperationException(); } - - @Override - public int freq() throws IOException { throw new UnsupportedOperationException(); } - - @Override - public int docID() { throw new UnsupportedOperationException(); } - - @Override - public int nextDoc() throws IOException { throw new UnsupportedOperationException(); } - - @Override - public int advance(int target) throws IOException { throw new UnsupportedOperationException(); } - - @Override - public long cost() { throw new UnsupportedOperationException(); } - }; + static final Scorer FAKESCORER = new FakeScorer(); } Index: lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (.../trunk) (revision 1643331) +++ lucene/misc/src/java/org/apache/lucene/uninverting/DocTermOrds.java (.../branches/lucene2878) (revision 1643331) @@ -25,7 +25,6 @@ import org.apache.lucene.codecs.PostingsFormat; // javadocs import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; @@ -618,7 +617,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { return termsEnum.docsAndPositions(liveDocs, reuse, flags); } Index: lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java =================================================================== --- lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (.../trunk) (revision 1643331) +++ lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (.../branches/lucene2878) (revision 1643331) @@ -17,13 +17,22 @@ * limitations under the License. */ - import java.io.IOException; -import java.util.Collection; +import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.FakeScorer; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.SimpleCollector; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.search.TopFieldCollector; +import org.apache.lucene.search.TopScoreDocCollector; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.PriorityQueue; @@ -85,56 +94,6 @@ private final GroupQueue groupQueue; private boolean groupCompetes; - private final static class FakeScorer extends Scorer { - - float score; - int doc; - - public FakeScorer() { - super(null); - } - - @Override - public float score() { - return score; - } - - @Override - public int freq() { - throw new UnsupportedOperationException(); // TODO: wtf does this class do? - } - - @Override - public int docID() { - return doc; - } - - @Override - public int advance(int target) { - throw new UnsupportedOperationException(); - } - - @Override - public int nextDoc() { - throw new UnsupportedOperationException(); - } - - @Override - public long cost() { - return 1; - } - - @Override - public Weight getWeight() { - throw new UnsupportedOperationException(); - } - - @Override - public Collection getChildren() { - throw new UnsupportedOperationException(); - } - } - private static final class OneGroup { LeafReaderContext readerContext; //int groupOrd; Index: lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java =================================================================== --- lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java (.../trunk) (revision 1643331) +++ lucene/misc/src/java/org/apache/lucene/uninverting/FieldCacheImpl.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,6 @@ * limitations under the License. */ -import java.io.IOException; -import java.io.PrintStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.WeakHashMap; - import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; @@ -51,6 +42,15 @@ import org.apache.lucene.util.packed.PackedInts; import org.apache.lucene.util.packed.PackedLongValues; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + /** * Expert: The default cache implementation, storing all values in memory. * A WeakHashMap is used for storage. Index: lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java =================================================================== --- lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java (.../trunk) (revision 1643331) +++ lucene/spatial/src/test/org/apache/lucene/spatial/prefix/DateNRStrategyTest.java (.../branches/lucene2878) (revision 1643331) @@ -27,7 +27,6 @@ import org.junit.Test; import java.io.IOException; -import java.text.ParseException; import java.util.Calendar; public class DateNRStrategyTest extends RandomSpatialOpStrategyTestCase { @@ -66,6 +65,12 @@ testOperationRandomShapes(SpatialOperation.Contains); } + @Test @Ignore("see LUCENE-5692") + @Repeat(iterations = ITERATIONS) + public void testDisjoint() throws IOException { + testOperationRandomShapes(SpatialOperation.IsDisjointTo); + } + @Test public void testWithinSame() throws IOException { final Calendar cal = tree.newCal(); Index: lucene/misc =================================================================== --- lucene/misc (.../trunk) (revision 1643331) +++ lucene/misc (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/misc ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/misc:r1638800-1643324 Index: lucene/grouping =================================================================== --- lucene/grouping (.../trunk) (revision 1643331) +++ lucene/grouping (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/grouping ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/grouping:r1638800-1642714 Index: lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsSegmentTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -20,12 +20,8 @@ //import java.io.*; //import java.nio.charset.StandardCharsets; -import java.io.IOException; -import java.io.PrintStream; - import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.TermState; @@ -42,6 +38,9 @@ import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.Util; +import java.io.IOException; +import java.io.PrintStream; + /** Iterates through terms in this field. */ public final class OrdsSegmentTermsEnum extends TermsEnum { @@ -937,7 +936,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits skipDocs, DocsEnum reuse, int flags) throws IOException { if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (.../branches/lucene2878) (revision 1643331) @@ -17,21 +17,11 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map.Entry; -import java.util.Map; - import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.bloom.FuzzySet.ContainsResult; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -51,6 +41,15 @@ import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.automaton.CompiledAutomaton; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + /** *

    * A {@link PostingsFormat} useful for low doc-frequency fields such as primary @@ -381,11 +380,10 @@ public long totalTermFreq() throws IOException { return delegate().totalTermFreq(); } - @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, - DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, + DocsEnum reuse, int flags) throws IOException { return delegate().docsAndPositions(liveDocs, reuse, flags); } @@ -394,6 +392,7 @@ throws IOException { return delegate().docs(liveDocs, reuse, flags); } + } @Override Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesProducer.java (.../branches/lucene2878) (revision 1643331) @@ -17,20 +17,11 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -67,6 +58,14 @@ import org.apache.lucene.util.packed.MonotonicBlockPackedReader; import org.apache.lucene.util.packed.PackedInts; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + /** * Reader for {@link MemoryDocValuesFormat} */ @@ -897,7 +896,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { throw new UnsupportedOperationException(); } } Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTOrdTermsReader.java (.../branches/lucene2878) (revision 1643331) @@ -17,22 +17,16 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.TreeMap; - +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; @@ -41,16 +35,16 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.automaton.ByteRunAutomaton; -import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.fst.BytesRefFSTEnum; import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; import org.apache.lucene.util.fst.FST; @@ -57,11 +51,16 @@ import org.apache.lucene.util.fst.Outputs; import org.apache.lucene.util.fst.PositiveIntOutputs; import org.apache.lucene.util.fst.Util; -import org.apache.lucene.codecs.BlockTermState; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.PostingsReaderBase; -import org.apache.lucene.codecs.CodecUtil; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; + /** * FST-based terms dictionary reader. * @@ -433,12 +432,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { - if (!hasPositions()) { - return null; - } - decodeMetaData(); - return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + return docs(liveDocs, reuse, flags); } // TODO: this can be achieved by making use of Util.getByOutput() Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (.../branches/lucene2878) (revision 1643331) @@ -27,7 +27,6 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat; // javadocs -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -50,7 +49,7 @@ import org.apache.lucene.util.automaton.RunAutomaton; import org.apache.lucene.util.automaton.Transition; -// TODO: +// TODO: // - build depth-N prefix hash? // - or: longer dense skip lists than just next byte? @@ -61,7 +60,7 @@ *

    WARNING: This is * exceptionally RAM intensive: it makes no effort to * compress the postings data, storing terms as separate - * byte[] and postings as separate int[], but as a result it + * byte[] and postings as separate int[], but as a result it * gives substantial increase in search performance. * *

    This postings format supports {@link TermsEnum#ord} @@ -88,7 +87,7 @@ public DirectPostingsFormat() { this(DEFAULT_MIN_SKIP_COUNT, DEFAULT_LOW_FREQ_CUTOFF); } - + /** minSkipCount is how many terms in a row must have the * same prefix before we put a skip pointer down. Terms * with docFreq <= lowFreqCutoff will use a single int[] @@ -99,7 +98,7 @@ this.minSkipCount = minSkipCount; this.lowFreqCutoff = lowFreqCutoff; } - + @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { return PostingsFormat.forName("Lucene50").fieldsConsumer(state); @@ -160,7 +159,7 @@ } return sizeInBytes; } - + @Override public Iterable getChildResources() { return Accountables.namedAccountables("field", fields); @@ -205,9 +204,14 @@ @Override public long ramBytesUsed() { return BASE_RAM_BYTES_USED + - ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) + + ((postings!=null) ? RamUsageEstimator.sizeOf(postings) : 0) + ((payloads!=null) ? RamUsageEstimator.sizeOf(payloads) : 0); } + + @Override + public Iterable getChildResources() { + return Collections.emptyList(); + } } // TODO: maybe specialize into prx/no-prx/no-frq cases? @@ -231,31 +235,36 @@ @Override public long ramBytesUsed() { - long sizeInBytes = BASE_RAM_BYTES_USED; - sizeInBytes += (docIDs!=null)? RamUsageEstimator.sizeOf(docIDs) : 0; - sizeInBytes += (freqs!=null)? RamUsageEstimator.sizeOf(freqs) : 0; - - if(positions != null) { - sizeInBytes += RamUsageEstimator.shallowSizeOf(positions); - for(int[] position : positions) { - sizeInBytes += (position!=null) ? RamUsageEstimator.sizeOf(position) : 0; - } - } - - if (payloads != null) { - sizeInBytes += RamUsageEstimator.shallowSizeOf(payloads); - for(byte[][] payload : payloads) { - if(payload != null) { - sizeInBytes += RamUsageEstimator.shallowSizeOf(payload); - for(byte[] pload : payload) { - sizeInBytes += (pload!=null) ? RamUsageEstimator.sizeOf(pload) : 0; - } - } - } - } - - return sizeInBytes; + long sizeInBytes = BASE_RAM_BYTES_USED; + sizeInBytes += (docIDs!=null)? RamUsageEstimator.sizeOf(docIDs) : 0; + sizeInBytes += (freqs!=null)? RamUsageEstimator.sizeOf(freqs) : 0; + + if(positions != null) { + sizeInBytes += RamUsageEstimator.shallowSizeOf(positions); + for(int[] position : positions) { + sizeInBytes += (position!=null) ? RamUsageEstimator.sizeOf(position) : 0; + } + } + + if (payloads != null) { + sizeInBytes += RamUsageEstimator.shallowSizeOf(payloads); + for(byte[][] payload : payloads) { + if(payload != null) { + sizeInBytes += RamUsageEstimator.shallowSizeOf(payload); + for(byte[] pload : payload) { + sizeInBytes += (pload!=null) ? RamUsageEstimator.sizeOf(pload) : 0; + } + } + } + } + + return sizeInBytes; } + + @Override + public Iterable getChildResources() { + return Collections.emptyList(); + } } private final byte[] termBytes; @@ -312,7 +321,7 @@ } terms = new TermAndSkip[numTerms]; termOffsets = new int[1+numTerms]; - + byte[] termBytes = new byte[1024]; this.minSkipCount = minSkipCount; @@ -324,7 +333,7 @@ BytesRef term; DocsEnum docsEnum = null; - DocsAndPositionsEnum docsAndPositionsEnum = null; + DocsEnum docsAndPositionsEnum = null; final TermsEnum termsEnum = termsIn.iterator(null); int termOffset = 0; @@ -411,7 +420,7 @@ } final int[] postings = scratch.get(); - + ent = new LowFreqTerm(postings, payloads, docFreq, (int) totalTermFreq); } else { final int[] docs = new int[docFreq]; @@ -523,7 +532,7 @@ sizeInBytes += ((skips!=null) ? RamUsageEstimator.sizeOf(skips) : 0); sizeInBytes += ((skipOffsets!=null) ? RamUsageEstimator.sizeOf(skipOffsets) : 0); sizeInBytes += ((sameCounts!=null) ? RamUsageEstimator.sizeOf(sameCounts) : 0); - + if(terms!=null) { sizeInBytes += RamUsageEstimator.shallowSizeOf(terms); for(TermAndSkip termAndSkip : terms) { @@ -530,11 +539,16 @@ sizeInBytes += (termAndSkip!=null) ? termAndSkip.ramBytesUsed() : 0; } } - + return sizeInBytes; } @Override + public Iterable getChildResources() { + return Collections.emptyList(); + } + + @Override public String toString() { return "DirectTerms(terms=" + terms.length + ",postings=" + sumDocFreq + ",positions=" + sumTotalTermFreq + ",docs=" + docCount + ")"; } @@ -546,7 +560,7 @@ int upto = termOffsets[ord]; final int termLen = termOffsets[1+ord] - upto; int otherUpto = other.offset; - + final int stop = upto + Math.min(termLen, other.length); while (upto < stop) { int diff = (termBytes[upto++] & 0xFF) - (otherBytes[otherUpto++] & 0xFF); @@ -554,7 +568,7 @@ return diff; } } - + // One is a prefix of the other, or, they are equal: return termLen - other.length; } @@ -706,7 +720,7 @@ public boolean hasPositions() { return hasPos; } - + @Override public boolean hasPayloads() { return hasPayloads; @@ -854,10 +868,13 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { // TODO: implement reuse // it's hairy! + if ((flags & DocsEnum.FLAG_POSITIONS) >= DocsEnum.FLAG_POSITIONS) + return docsAndPositions(liveDocs, reuse, flags); + if (terms[termOrd] instanceof LowFreqTerm) { final int[] postings = ((LowFreqTerm) terms[termOrd]).postings; if (hasFreq) { @@ -927,7 +944,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) { if (!hasPos) { return null; } @@ -1202,7 +1219,7 @@ // if (DEBUG) { // System.out.println(" term=" + new BytesRef(termBytes, termOffset, termLength).utf8ToString() + " skips=" + Arrays.toString(skips)); // } - + assert termOrd < state.changeOrd; assert stateUpto <= termLength: "term.length=" + termLength + "; stateUpto=" + stateUpto; @@ -1335,7 +1352,7 @@ compiledAutomaton.automaton.initTransition(nextState, states[stateUpto].transition); states[stateUpto].transitionUpto = -1; states[stateUpto].transitionMax = -1; - + if (stateUpto == termLength) { // if (DEBUG) { // System.out.println(" term ends after push"); @@ -1455,6 +1472,8 @@ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) { // TODO: implement reuse // it's hairy! + if ((flags & DocsEnum.FLAG_POSITIONS) >= DocsEnum.FLAG_POSITIONS) + return docsAndPositions(liveDocs, reuse, flags); if (terms[termOrd] instanceof LowFreqTerm) { final int[] postings = ((LowFreqTerm) terms[termOrd]).postings; @@ -1484,7 +1503,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) { if (!hasPos) { return null; } @@ -1572,12 +1591,42 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int target) throws IOException { // Linear scan, but this is low-freq term so it won't // be costly: return slowAdvance(target); } - + @Override public long cost() { return postings.length; @@ -1640,12 +1689,42 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int target) throws IOException { // Linear scan, but this is low-freq term so it won't // be costly: return slowAdvance(target); } - + @Override public long cost() { return postings.length / 2; @@ -1687,7 +1766,7 @@ // System.out.println(" nextDoc freq=" + freq + " upto=" + upto + " vs " + postings.length); // } if (liveDocs == null) { - if (upto < postings.length) { + if (upto < postings.length) { freq = postings[upto+1]; assert freq > 0; return postings[upto]; @@ -1724,12 +1803,42 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int target) throws IOException { // Linear scan, but this is low-freq term so it won't // be costly: return slowAdvance(target); } - + @Override public long cost() { // TODO: could do a better estimate @@ -1737,7 +1846,7 @@ } } - private final static class LowFreqDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final static class LowFreqDocsAndPositionsEnum extends DocsEnum { private int[] postings; private final Bits liveDocs; private final int posMult; @@ -1748,6 +1857,7 @@ private int docID; private int freq; private int skipPositions; + private int pos; private int startOffset; private int endOffset; private int lastPayloadOffset; @@ -1772,10 +1882,11 @@ } } - public DocsAndPositionsEnum reset(int[] postings, byte[] payloadBytes) { + public DocsEnum reset(int[] postings, byte[] payloadBytes) { this.postings = postings; upto = 0; skipPositions = 0; + pos = -1; startOffset = -1; endOffset = -1; docID = -1; @@ -1786,6 +1897,7 @@ @Override public int nextDoc() { + pos = -1; if (hasPayloads) { for(int i=0;i 0; + if (skipPositions <= 0) + return NO_MORE_POSITIONS; skipPositions--; - final int pos = postings[upto++]; + pos = postings[upto++]; if (hasOffsets) { startOffset = postings[upto++]; endOffset = postings[upto++]; @@ -1858,6 +1971,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() { return startOffset; } @@ -1883,7 +2006,7 @@ return null; } } - + @Override public long cost() { // TODO: could do a better estimate @@ -1956,6 +2079,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int advance(int target) { /* upto++; @@ -2062,7 +2215,7 @@ return docID = docIDs[upto]; } } - + @Override public long cost() { return docIDs.length; @@ -2070,7 +2223,7 @@ } // TODO: specialize offsets and not - private final static class HighFreqDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final static class HighFreqDocsAndPositionsEnum extends DocsEnum { private int[] docIDs; private int[] freqs; private int[][] positions; @@ -2105,7 +2258,7 @@ return liveDocs; } - public DocsAndPositionsEnum reset(int[] docIDs, int[] freqs, int[][] positions, byte[][][] payloads) { + public DocsEnum reset(int[] docIDs, int[] freqs, int[][] positions, byte[][][] payloads) { this.docIDs = docIDs; this.freqs = freqs; this.positions = positions; @@ -2119,7 +2272,7 @@ upto++; if (liveDocs == null) { if (upto < docIDs.length) { - posUpto = -posJump; + posUpto = -posJump; curPositions = positions[upto]; return docID = docIDs[upto]; } @@ -2150,10 +2303,22 @@ @Override public int nextPosition() { posUpto += posJump; + if (posUpto >= curPositions.length) + return NO_MORE_POSITIONS; return curPositions[posUpto]; } @Override + public int startPosition() throws IOException { + return curPositions[posUpto]; + } + + @Override + public int endPosition() throws IOException { + return startPosition(); + } + + @Override public int startOffset() { if (hasOffsets) { return curPositions[posUpto+1]; @@ -2300,7 +2465,7 @@ return payload; } } - + @Override public long cost() { return docIDs.length; Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (.../branches/lucene2878) (revision 1643331) @@ -17,21 +17,16 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; -import java.util.TreeMap; - +import org.apache.lucene.codecs.BlockTermState; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.FieldsProducer; +import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.TermState; @@ -39,26 +34,30 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.automaton.ByteRunAutomaton; -import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountables; +import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.fst.BytesRefFSTEnum; import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput; import org.apache.lucene.util.fst.FST; import org.apache.lucene.util.fst.Outputs; import org.apache.lucene.util.fst.Util; -import org.apache.lucene.codecs.BlockTermState; -import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.codecs.PostingsReaderBase; -import org.apache.lucene.codecs.CodecUtil; +import java.io.IOException; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.TreeMap; + /** * FST-based terms dictionary reader. * @@ -296,12 +295,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { - if (!hasPositions()) { - return null; - } - decodeMetaData(); - return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + return docs(liveDocs, reuse, flags); } @Override Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,7 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.codecs.FieldsConsumer; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -32,8 +29,10 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; +import java.io.IOException; + class SimpleTextFieldsWriter extends FieldsConsumer { - + private IndexOutput out; private final BytesRefBuilder scratch = new BytesRefBuilder(); private final SegmentWriteState writeState; @@ -81,10 +80,10 @@ if (hasPositions) { if (hasPayloads) { - flags = flags | DocsAndPositionsEnum.FLAG_PAYLOADS; + flags = flags | DocsEnum.FLAG_PAYLOADS; } if (hasOffsets) { - flags = flags | DocsAndPositionsEnum.FLAG_OFFSETS; + flags = flags | DocsEnum.FLAG_OFFSETS; } } else { if (hasFreqs) { @@ -93,7 +92,6 @@ } TermsEnum termsEnum = terms.iterator(null); - DocsAndPositionsEnum posEnum = null; DocsEnum docsEnum = null; // for each term in field @@ -104,8 +102,7 @@ } if (hasPositions) { - posEnum = termsEnum.docsAndPositions(null, posEnum, flags); - docsEnum = posEnum; + docsEnum = termsEnum.docsAndPositions(null, docsEnum, flags); } else { docsEnum = termsEnum.docs(null, docsEnum, flags); } @@ -154,7 +151,7 @@ // for each pos in field+term+doc for(int i=0;i= startOffset; assert startOffset >= lastStartOffset: "startOffset=" + startOffset + " lastStartOffset=" + lastStartOffset; lastStartOffset = startOffset; @@ -174,7 +171,7 @@ newline(); } - BytesRef payload = posEnum.getPayload(); + BytesRef payload = docsEnum.getPayload(); if (payload != null && payload.length > 0) { assert payload.length != 0; Index: lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java =================================================================== --- lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java (.../trunk) (revision 1643331) +++ lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/NumberRangePrefixTree.java (.../branches/lucene2878) (revision 1643331) @@ -232,35 +232,21 @@ public Shape toRangeShape(Shape start, Shape end) { if (!(start instanceof LevelledValue && end instanceof LevelledValue)) throw new IllegalArgumentException("Must pass "+LevelledValue.class+" but got "+start.getClass()); - LevelledValue startLV = (LevelledValue) start; - LevelledValue endLV = (LevelledValue) end; - //note: this normalization/optimization process is actually REQUIRED based on assumptions elsewhere. - //Normalize start & end - startLV = startLV.getLVAtLevel(truncateStartVals(startLV, 0)); // chops off trailing min-vals (zeroes) - endLV = endLV.getLVAtLevel(truncateEndVals(endLV, 0)); // chops off trailing max-vals - //Optimize to just start or end if it's equivalent, e.g. April to April 1st is April 1st. - int cmp = comparePrefixLV(startLV, endLV); + LevelledValue minLV = (LevelledValue) start; + LevelledValue maxLV = (LevelledValue) end; + if (minLV.equals(maxLV)) + return minLV; + //Optimize precision of the range, e.g. April 1st to April 30th is April. + minLV = minLV.getLVAtLevel(truncateStartVals(minLV, 0)); + maxLV = maxLV.getLVAtLevel(truncateEndVals(maxLV, 0)); + int cmp = comparePrefixLV(minLV, maxLV); if (cmp > 0) { throw new IllegalArgumentException("Wrong order: "+start+" TO "+end); } - if (cmp == 0) {//one is a prefix of the other - if (startLV.getLevel() == endLV.getLevel()) { - //same - return startLV; - } else if (endLV.getLevel() > startLV.getLevel()) { - // e.g. April to April 1st - if (truncateStartVals(endLV, startLV.getLevel()) == startLV.getLevel()) { - return endLV; + if (cmp == 0 && minLV.getLevel() == maxLV.getLevel()) + return minLV; + return new NRShape(minLV, maxLV); } - } else {//minLV level > maxLV level - // e.g. April 30 to April - if (truncateEndVals(startLV, endLV.getLevel()) == endLV.getLevel()) { - return startLV; - } - } - } - return new NRShape(startLV, endLV); - } /** From lv.getLevel on up, it returns the first Level seen with val != 0. It doesn't check past endLevel. */ private int truncateStartVals(LevelledValue lv, int endLevel) { Index: lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsIntersectTermsEnum.java (.../branches/lucene2878) (revision 1643331) @@ -17,10 +17,7 @@ * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.TermState; @@ -35,6 +32,8 @@ import org.apache.lucene.util.automaton.RunAutomaton; import org.apache.lucene.util.fst.FST; +import java.io.IOException; + // NOTE: cannot seek! final class OrdsIntersectTermsEnum extends TermsEnum { final IndexInput in; @@ -209,7 +208,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits skipDocs, DocsEnum reuse, int flags) throws IOException { if (fr.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (.../branches/lucene2878) (revision 1643331) @@ -30,13 +30,12 @@ import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.TermStats; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; @@ -317,7 +316,7 @@ long sumTotalTermFreq = 0; long sumDocFreq = 0; DocsEnum docsEnum = null; - DocsAndPositionsEnum posEnum = null; + DocsEnum posEnum = null; int enumFlags; IndexOptions indexOptions = fieldInfo.getIndexOptions(); @@ -332,15 +331,16 @@ enumFlags = DocsEnum.FLAG_FREQS; } else if (writeOffsets == false) { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS; - } else { - enumFlags = 0; + enumFlags = DocsEnum.FLAG_PAYLOADS; } + else { + enumFlags = DocsEnum.FLAG_POSITIONS; + } } else { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS | DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_PAYLOADS | DocsEnum.FLAG_OFFSETS; } else { - enumFlags = DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_OFFSETS; } } @@ -539,14 +539,44 @@ public int freq() { return freq; } - + @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override + public int startPosition() throws IOException { + return -1; + } + + @Override + public int endPosition() throws IOException { + return -1; + } + + @Override + public int startOffset() throws IOException { + return -1; + } + + @Override + public int endOffset() throws IOException { + return -1; + } + + @Override + public BytesRef getPayload() throws IOException { + return null; + } + + @Override public long cost() { return numDocs; } } - private final static class FSTDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final static class FSTDocsAndPositionsEnum extends DocsEnum { private final boolean storePayloads; private byte[] buffer = new byte[16]; private final ByteArrayDataInput in = new ByteArrayDataInput(buffer); @@ -660,7 +690,8 @@ @Override public int nextPosition() { //System.out.println(" nextPos storePayloads=" + storePayloads + " this=" + this); - assert posPending > 0; + if (posPending <= 0) + return NO_MORE_POSITIONS; posPending--; if (!storePayloads) { pos += in.readVInt(); @@ -695,6 +726,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() { return startOffset; } @@ -802,6 +843,10 @@ @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) { + + if ((flags & DocsEnum.FLAG_POSITIONS) >= DocsEnum.FLAG_POSITIONS) + return docsAndPositions(liveDocs, reuse, flags); + decodeMetaData(); FSTDocsEnum docsEnum; @@ -817,7 +862,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) { boolean hasOffsets = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (.../branches/lucene2878) (revision 1643331) @@ -29,8 +29,8 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsReaderBase; import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexOptions; @@ -659,7 +659,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (.../branches/lucene2878) (revision 1643331) @@ -25,7 +25,6 @@ import java.util.TreeMap; import org.apache.lucene.codecs.TermVectorsReader; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexFileNames; @@ -59,15 +58,15 @@ public class SimpleTextTermVectorsReader extends TermVectorsReader { private static final long BASE_RAM_BYTES_USED = - RamUsageEstimator.shallowSizeOfInstance(SimpleTextTermVectorsReader.class) - + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class) - + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class); + RamUsageEstimator.shallowSizeOfInstance(SimpleTextTermVectorsReader.class) + + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class) + + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class); private long offsets[]; /* docid -> offset in .vec file */ private IndexInput in; private BytesRefBuilder scratch = new BytesRefBuilder(); private CharsRefBuilder scratchUTF16 = new CharsRefBuilder(); - + public SimpleTextTermVectorsReader(Directory directory, SegmentInfo si, IOContext context) throws IOException { boolean success = false; try { @@ -82,15 +81,15 @@ } readIndex(si.getDocCount()); } - + // used by clone SimpleTextTermVectorsReader(long offsets[], IndexInput in) { this.offsets = offsets; this.in = in; } - - // we don't actually write a .tvx-like index, instead we read the - // vectors file in entirety up-front and save the offsets + + // we don't actually write a .tvx-like index, instead we read the + // vectors file in entirety up-front and save the offsets // so we can seek to the data later. private void readIndex(int maxDoc) throws IOException { ChecksumIndexInput input = new BufferedChecksumIndexInput(in); @@ -106,7 +105,7 @@ SimpleTextUtil.checkFooter(input); assert upto == offsets.length; } - + @Override public Fields get(int doc) throws IOException { SortedMap fields = new TreeMap<>(); @@ -122,30 +121,30 @@ assert StringHelper.startsWith(scratch.get(), FIELD); // skip fieldNumber: parseIntAt(FIELD.length); - + readLine(); assert StringHelper.startsWith(scratch.get(), FIELDNAME); String fieldName = readString(FIELDNAME.length, scratch); - + readLine(); assert StringHelper.startsWith(scratch.get(), FIELDPOSITIONS); boolean positions = Boolean.parseBoolean(readString(FIELDPOSITIONS.length, scratch)); - + readLine(); assert StringHelper.startsWith(scratch.get(), FIELDOFFSETS); boolean offsets = Boolean.parseBoolean(readString(FIELDOFFSETS.length, scratch)); - + readLine(); assert StringHelper.startsWith(scratch.get(), FIELDPAYLOADS); boolean payloads = Boolean.parseBoolean(readString(FIELDPAYLOADS.length, scratch)); - + readLine(); assert StringHelper.startsWith(scratch.get(), FIELDTERMCOUNT); int termCount = parseIntAt(FIELDTERMCOUNT.length); - + SimpleTVTerms terms = new SimpleTVTerms(offsets, positions, payloads); fields.put(fieldName, terms); - + BytesRefBuilder term = new BytesRefBuilder(); for (int j = 0; j < termCount; j++) { readLine(); @@ -154,14 +153,14 @@ term.grow(termLength); term.setLength(termLength); System.arraycopy(scratch.bytes(), TERMTEXT.length, term.bytes(), 0, termLength); - + SimpleTVPostings postings = new SimpleTVPostings(); terms.terms.put(term.toBytesRef(), postings); - + readLine(); assert StringHelper.startsWith(scratch.get(), TERMFREQ); postings.freq = parseIntAt(TERMFREQ.length); - + if (positions || offsets) { if (positions) { postings.positions = new int[postings.freq]; @@ -169,12 +168,12 @@ postings.payloads = new BytesRef[postings.freq]; } } - + if (offsets) { postings.startOffsets = new int[postings.freq]; postings.endOffsets = new int[postings.freq]; } - + for (int k = 0; k < postings.freq; k++) { if (positions) { readLine(); @@ -192,12 +191,12 @@ } } } - + if (offsets) { readLine(); assert StringHelper.startsWith(scratch.get(), STARTOFFSET); postings.startOffsets[k] = parseIntAt(STARTOFFSET.length); - + readLine(); assert StringHelper.startsWith(scratch.get(), ENDOFFSET); postings.endOffsets[k] = parseIntAt(ENDOFFSET.length); @@ -216,11 +215,11 @@ } return new SimpleTextTermVectorsReader(offsets, in.clone()); } - + @Override public void close() throws IOException { try { - IOUtils.close(in); + IOUtils.close(in); } finally { in = null; offsets = null; @@ -230,20 +229,20 @@ private void readLine() throws IOException { SimpleTextUtil.readLine(in, scratch); } - + private int parseIntAt(int offset) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), offset, scratch.length()-offset); return ArrayUtil.parseInt(scratchUTF16.chars(), 0, scratchUTF16.length()); } - + private String readString(int offset, BytesRefBuilder scratch) { scratchUTF16.copyUTF8Bytes(scratch.bytes(), offset, scratch.length()-offset); return scratchUTF16.toString(); } - + private class SimpleTVFields extends Fields { private final SortedMap fields; - + SimpleTVFields(SortedMap fields) { this.fields = fields; } @@ -263,13 +262,13 @@ return fields.size(); } } - + private static class SimpleTVTerms extends Terms { final SortedMap terms; final boolean hasOffsets; final boolean hasPositions; final boolean hasPayloads; - + SimpleTVTerms(boolean hasOffsets, boolean hasPositions, boolean hasPayloads) { this.hasOffsets = hasOffsets; this.hasPositions = hasPositions; @@ -276,7 +275,7 @@ this.hasPayloads = hasPayloads; terms = new TreeMap<>(); } - + @Override public TermsEnum iterator(TermsEnum reuse) throws IOException { // TODO: reuse @@ -317,13 +316,13 @@ public boolean hasPositions() { return hasPositions; } - + @Override public boolean hasPayloads() { return hasPayloads; } } - + private static class SimpleTVPostings { private int freq; private int positions[]; @@ -331,17 +330,17 @@ private int endOffsets[]; private BytesRef payloads[]; } - + private static class SimpleTVTermsEnum extends TermsEnum { SortedMap terms; Iterator> iterator; Map.Entry current; - + SimpleTVTermsEnum(SortedMap terms) { this.terms = terms; this.iterator = terms.entrySet().iterator(); } - + @Override public SeekStatus seekCeil(BytesRef text) throws IOException { iterator = terms.tailMap(text).entrySet().iterator(); @@ -396,7 +395,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { SimpleTVPostings postings = current.getValue(); if (postings.positions == null && postings.startOffsets == null) { return null; @@ -407,7 +406,7 @@ return e; } } - + // note: these two enum classes are exactly like the Default impl... private static class SimpleTVDocsEnum extends DocsEnum { private boolean didNext; @@ -422,6 +421,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int docID() { return doc; } @@ -447,14 +476,14 @@ this.doc = -1; didNext = false; } - + @Override public long cost() { return 1; } } - - private static class SimpleTVDocsAndPositionsEnum extends DocsAndPositionsEnum { + + private static class SimpleTVDocsAndPositionsEnum extends DocsEnum { private boolean didNext; private int doc = -1; private int nextPos; @@ -512,11 +541,13 @@ @Override public int nextPosition() { - assert (positions != null && nextPos < positions.length) || - startOffsets != null && nextPos < startOffsets.length; if (positions != null) { + if (nextPos >= positions.length) + return NO_MORE_POSITIONS; return positions[nextPos++]; } else { + if (nextPos >= startOffsets.length) + return NO_MORE_POSITIONS; nextPos++; return -1; } @@ -523,6 +554,16 @@ } @Override + public int startPosition() throws IOException { + return positions[nextPos-1]; + } + + @Override + public int endPosition() throws IOException { + return positions[nextPos-1]; + } + + @Override public int startOffset() { if (startOffsets == null) { return -1; @@ -539,7 +580,7 @@ return endOffsets[nextPos-1]; } } - + @Override public long cost() { return 1; @@ -550,7 +591,7 @@ public long ramBytesUsed() { return BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(offsets); } - + @Override public String toString() { return getClass().getSimpleName(); Index: lucene/spatial =================================================================== --- lucene/spatial (.../trunk) (revision 1643331) +++ lucene/spatial (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/spatial ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/spatial:r1638800-1642714 Index: lucene/BUILD.txt =================================================================== --- lucene/BUILD.txt (.../trunk) (revision 1643331) +++ lucene/BUILD.txt (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/BUILD.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/BUILD.txt:r1638800-1642144 Index: solr/site =================================================================== --- solr/site (.../trunk) (revision 1643331) +++ solr/site (.../branches/lucene2878) (revision 1643331) Property changes on: solr/site ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/site:r1638800-1642144 Index: solr/SYSTEM_REQUIREMENTS.txt =================================================================== --- solr/SYSTEM_REQUIREMENTS.txt (.../trunk) (revision 1643331) +++ solr/SYSTEM_REQUIREMENTS.txt (.../branches/lucene2878) (revision 1643331) Property changes on: solr/SYSTEM_REQUIREMENTS.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/SYSTEM_REQUIREMENTS.txt:r1638800-1642144 Index: lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingPostingsFormat.java (.../branches/lucene2878) (revision 1643331) @@ -17,14 +17,10 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Iterator; - import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.index.AssertingLeafReader; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -38,6 +34,9 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.TestUtil; +import java.io.IOException; +import java.util.Iterator; + /** * Just like the default postings format but with additional asserts. */ @@ -159,7 +158,6 @@ termsEnum = terms.iterator(termsEnum); BytesRefBuilder lastTerm = null; DocsEnum docsEnum = null; - DocsAndPositionsEnum posEnum = null; boolean hasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; boolean hasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; @@ -190,13 +188,12 @@ docsEnum = termsEnum.docs(null, docsEnum, flags); } else { if (hasPayloads) { - flags |= DocsAndPositionsEnum.FLAG_PAYLOADS; + flags |= DocsEnum.FLAG_PAYLOADS; } if (hasOffsets) { - flags = flags | DocsAndPositionsEnum.FLAG_OFFSETS; + flags = flags | DocsEnum.FLAG_OFFSETS; } - posEnum = termsEnum.docsAndPositions(null, posEnum, flags); - docsEnum = posEnum; + docsEnum = termsEnum.docsAndPositions(null, docsEnum, flags); } assert docsEnum != null : "termsEnum=" + termsEnum + " hasPositions=" + hasPositions; @@ -218,13 +215,13 @@ int lastPos = -1; int lastStartOffset = -1; for(int i=0;i= lastPos: "pos=" + pos + " vs lastPos=" + lastPos + " i=" + i + " freq=" + freq; lastPos = pos; if (hasOffsets) { - int startOffset = posEnum.startOffset(); - int endOffset = posEnum.endOffset(); + int startOffset = docsEnum.startOffset(); + int endOffset = docsEnum.endOffset(); assert endOffset >= startOffset; assert startOffset >= lastStartOffset; lastStartOffset = startOffset; Index: lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (.../branches/lucene2878) (revision 1643331) @@ -33,7 +33,6 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.TermStats; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; @@ -252,7 +251,6 @@ long sumTotalTermFreq = 0; long sumDocFreq = 0; DocsEnum docsEnum = null; - DocsAndPositionsEnum posEnum = null; int enumFlags; IndexOptions indexOptions = fieldInfo.getIndexOptions(); @@ -267,15 +265,15 @@ enumFlags = DocsEnum.FLAG_FREQS; } else if (writeOffsets == false) { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS; + enumFlags = DocsEnum.FLAG_PAYLOADS; } else { enumFlags = 0; } } else { if (writePayloads) { - enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS | DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_PAYLOADS | DocsEnum.FLAG_OFFSETS; } else { - enumFlags = DocsAndPositionsEnum.FLAG_OFFSETS; + enumFlags = DocsEnum.FLAG_OFFSETS; } } @@ -285,15 +283,8 @@ break; } RAMPostingsWriterImpl postingsWriter = termsConsumer.startTerm(term); + docsEnum = termsEnum.docs(null, docsEnum, enumFlags); - if (writePositions) { - posEnum = termsEnum.docsAndPositions(null, posEnum, enumFlags); - docsEnum = posEnum; - } else { - docsEnum = termsEnum.docs(null, docsEnum, enumFlags); - posEnum = null; - } - int docFreq = 0; long totalTermFreq = 0; while (true) { @@ -315,13 +306,13 @@ postingsWriter.startDoc(docID, freq); if (writePositions) { for (int i=0;i= current.positions.length) + return NO_MORE_POSITIONS; + return current.positions[posUpto++]; } - // TODO: override bulk read, for better perf @Override - public int nextDoc() { - while(true) { - upto++; - if (upto < ramTerm.docs.size()) { - current = ramTerm.docs.get(upto); - if (liveDocs == null || liveDocs.get(current.docID)) { - posUpto = 0; - return current.docID; - } - } else { - return NO_MORE_DOCS; - } - } + public int startPosition() throws IOException { + return current.positions[posUpto-1]; } @Override - public int freq() throws IOException { - return current.positions.length; + public int endPosition() throws IOException { + return current.positions[posUpto-1]; } @Override - public int docID() { - return current.docID; - } - - @Override - public int nextPosition() { - return current.positions[posUpto++]; - } - - @Override public int startOffset() { return -1; } Index: lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/search/AssertingWeight.java (.../branches/lucene2878) (revision 1643331) @@ -17,12 +17,12 @@ * limitations under the License. */ +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.Bits; + import java.io.IOException; import java.util.Random; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.Bits; - class AssertingWeight extends Weight { static Weight wrap(Random random, Weight other) { @@ -60,18 +60,18 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { // if the caller asks for in-order scoring or if the weight does not support // out-of order scoring then collection will have to happen in-order. - final Scorer inScorer = in.scorer(context, acceptDocs); + final Scorer inScorer = in.scorer(context, flags, acceptDocs); return AssertingScorer.wrap(new Random(random.nextLong()), inScorer); } @Override - public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, Bits acceptDocs) throws IOException { + public BulkScorer bulkScorer(LeafReaderContext context, boolean scoreDocsInOrder, int flags, Bits acceptDocs) throws IOException { // if the caller asks for in-order scoring or if the weight does not support // out-of order scoring then collection will have to happen in-order. - BulkScorer inScorer = in.bulkScorer(context, scoreDocsInOrder, acceptDocs); + BulkScorer inScorer = in.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); if (inScorer == null) { return null; } @@ -83,7 +83,7 @@ } else if (random.nextBoolean()) { // Let super wrap this.scorer instead, so we use // AssertingScorer: - inScorer = super.bulkScorer(context, scoreDocsInOrder, acceptDocs); + inScorer = super.bulkScorer(context, scoreDocsInOrder, flags, acceptDocs); } if (scoreDocsInOrder == false && random.nextBoolean()) { Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseTermVectorsFormatTestCase.java (.../branches/lucene2878) (revision 1643331) @@ -17,16 +17,7 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicReference; - +import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; @@ -49,7 +40,15 @@ import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.TestUtil; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; /** * Base class aiming at testing {@link TermVectorsFormat term vectors formats}. @@ -415,7 +414,7 @@ // to test reuse private final ThreadLocal termsEnum = new ThreadLocal<>(); private final ThreadLocal docsEnum = new ThreadLocal<>(); - private final ThreadLocal docsAndPositionsEnum = new ThreadLocal<>(); + private final ThreadLocal docsAndPositionsEnum = new ThreadLocal<>(); protected void assertEquals(RandomTokenStream tk, FieldType ft, Terms terms) throws IOException { assertEquals(1, terms.getDocCount()); @@ -453,7 +452,7 @@ this.docsEnum.set(docsEnum); bits.clear(0); - DocsAndPositionsEnum docsAndPositionsEnum = termsEnum.docsAndPositions(bits, random().nextBoolean() ? null : this.docsAndPositionsEnum.get()); + DocsEnum docsAndPositionsEnum = termsEnum.docsAndPositions(bits, random().nextBoolean() ? null : this.docsEnum.get()); assertEquals(ft.storeTermVectorOffsets() || ft.storeTermVectorPositions(), docsAndPositionsEnum != null); if (docsAndPositionsEnum != null) { assertEquals(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); @@ -519,7 +518,7 @@ } assertEquals(DocsEnum.NO_MORE_DOCS, docsAndPositionsEnum.nextDoc()); } - this.docsAndPositionsEnum.set(docsAndPositionsEnum); + this.docsEnum.set(docsAndPositionsEnum); } assertNull(termsEnum.next()); for (int i = 0; i < 5; ++i) { Index: lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/index/BaseDocValuesFormatTestCase.java (.../branches/lucene2878) (revision 1643331) @@ -17,8 +17,6 @@ * limitations under the License. */ -import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -61,6 +59,8 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.TestUtil; +import static org.apache.lucene.index.SortedSetDocValues.NO_MORE_ORDS; + /** * Abstract class to do basic tests for a docvalues format. * NOTE: This test focuses on the docvalues impl, nothing else. @@ -2083,7 +2083,7 @@ ); } } - + public void testSortedNumericsMultipleValuesVsStoredFields() throws Exception { assumeTrue("Codec does not support SORTED_NUMERIC", codecSupportsSortedNumeric()); int numIterations = atLeast(1); Index: lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (.../branches/lucene2878) (revision 1643331) @@ -121,7 +121,7 @@ /** Given the same random seed this always enumerates the * same random postings */ - private static class SeedPostings extends DocsAndPositionsEnum { + private static class SeedPostings extends DocsEnum { // Used only to generate docIDs; this way if you pull w/ // or w/o positions you get the same docID sequence: private final Random docRandom; @@ -233,7 +233,9 @@ posUpto = freq; return 0; } - assert posUpto < freq; + //assert posUpto < freq; + if (posUpto >= freq) + return NO_MORE_POSITIONS; if (posUpto == 0 && random.nextBoolean()) { // Sometimes index pos = 0 @@ -270,8 +272,18 @@ posUpto++; return pos; } - + @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() { return startOffset; } @@ -650,7 +662,7 @@ } @Override - public final DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public final DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { if (liveDocs != null) { throw new IllegalArgumentException("liveDocs must be null"); } @@ -657,10 +669,10 @@ if (maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { return null; } - if ((flags & DocsAndPositionsEnum.FLAG_OFFSETS) != 0 && maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { + if ((flags & DocsEnum.FLAG_OFFSETS) == DocsEnum.FLAG_OFFSETS && maxAllowed.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { return null; } - if ((flags & DocsAndPositionsEnum.FLAG_PAYLOADS) != 0 && allowPayloads == false) { + if ((flags & DocsEnum.FLAG_PAYLOADS) == DocsEnum.FLAG_PAYLOADS && allowPayloads == false) { return null; } return getSeedPostings(current.getKey().utf8ToString(), current.getValue().seed, false, maxAllowed, allowPayloads); @@ -746,7 +758,6 @@ private static class ThreadState { // Only used with REUSE option: public DocsEnum reuseDocsEnum; - public DocsAndPositionsEnum reuseDocsAndPositionsEnum; } private void verifyEnum(ThreadState threadState, @@ -812,7 +823,6 @@ DocsEnum prevDocsEnum = null; DocsEnum docsEnum; - DocsAndPositionsEnum docsAndPositionsEnum; if (!doCheckPositions) { if (allowPositions && random().nextInt(10) == 7) { @@ -819,24 +829,23 @@ // 10% of the time, even though we will not check positions, pull a DocsAndPositions enum if (options.contains(Option.REUSE_ENUMS) && random().nextInt(10) < 9) { - prevDocsEnum = threadState.reuseDocsAndPositionsEnum; + prevDocsEnum = threadState.reuseDocsEnum; } - int flags = 0; + int flags = DocsEnum.FLAG_POSITIONS; if (alwaysTestMax || random().nextBoolean()) { - flags |= DocsAndPositionsEnum.FLAG_OFFSETS; + flags |= DocsEnum.FLAG_OFFSETS; } if (alwaysTestMax || random().nextBoolean()) { - flags |= DocsAndPositionsEnum.FLAG_PAYLOADS; + flags |= DocsEnum.FLAG_PAYLOADS; } if (VERBOSE) { - System.out.println(" get DocsAndPositionsEnum (but we won't check positions) flags=" + flags); + System.out.println(" get DocsEnum (but we won't check positions) flags=" + flags); } - threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum, flags); - docsEnum = threadState.reuseDocsAndPositionsEnum; - docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum; + threadState.reuseDocsEnum = termsEnum.docsAndPositions(liveDocs, prevDocsEnum, flags); + docsEnum = threadState.reuseDocsEnum; } else { if (VERBOSE) { System.out.println(" get DocsEnum"); @@ -846,28 +855,26 @@ } threadState.reuseDocsEnum = termsEnum.docs(liveDocs, prevDocsEnum, doCheckFreqs ? DocsEnum.FLAG_FREQS : DocsEnum.FLAG_NONE); docsEnum = threadState.reuseDocsEnum; - docsAndPositionsEnum = null; } } else { if (options.contains(Option.REUSE_ENUMS) && random().nextInt(10) < 9) { - prevDocsEnum = threadState.reuseDocsAndPositionsEnum; + prevDocsEnum = threadState.reuseDocsEnum; } - int flags = 0; + int flags = DocsEnum.FLAG_POSITIONS; if (alwaysTestMax || doCheckOffsets || random().nextInt(3) == 1) { - flags |= DocsAndPositionsEnum.FLAG_OFFSETS; + flags |= DocsEnum.FLAG_OFFSETS; } if (alwaysTestMax || doCheckPayloads|| random().nextInt(3) == 1) { - flags |= DocsAndPositionsEnum.FLAG_PAYLOADS; + flags |= DocsEnum.FLAG_PAYLOADS; } if (VERBOSE) { - System.out.println(" get DocsAndPositionsEnum flags=" + flags); + System.out.println(" get DocsEnum flags=" + flags); } - threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum, flags); - docsEnum = threadState.reuseDocsAndPositionsEnum; - docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum; + threadState.reuseDocsEnum = termsEnum.docsAndPositions(liveDocs, prevDocsEnum, flags); + docsEnum = threadState.reuseDocsEnum; } assertNotNull("null DocsEnum", docsEnum); @@ -1009,7 +1016,7 @@ if (VERBOSE) { System.out.println(" now nextPosition to " + pos); } - assertEquals("position is wrong", pos, docsAndPositionsEnum.nextPosition()); + assertEquals("position is wrong", pos, docsEnum.nextPosition()); if (doCheckPayloads) { BytesRef expectedPayload = expected.getPayload(); @@ -1018,9 +1025,9 @@ System.out.println(" now check expectedPayload length=" + (expectedPayload == null ? 0 : expectedPayload.length)); } if (expectedPayload == null || expectedPayload.length == 0) { - assertNull("should not have payload", docsAndPositionsEnum.getPayload()); + assertNull("should not have payload", docsEnum.getPayload()); } else { - BytesRef payload = docsAndPositionsEnum.getPayload(); + BytesRef payload = docsEnum.getPayload(); assertNotNull("should have payload but doesn't", payload); assertEquals("payload length is wrong", expectedPayload.length, payload.length); @@ -1032,7 +1039,7 @@ // make a deep copy payload = BytesRef.deepCopyOf(payload); - assertEquals("2nd call to getPayload returns something different!", payload, docsAndPositionsEnum.getPayload()); + assertEquals("2nd call to getPayload returns something different!", payload, docsEnum.getPayload()); } } else { if (VERBOSE) { @@ -1046,8 +1053,8 @@ if (VERBOSE) { System.out.println(" now check offsets: startOff=" + expected.startOffset() + " endOffset=" + expected.endOffset()); } - assertEquals("startOffset is wrong", expected.startOffset(), docsAndPositionsEnum.startOffset()); - assertEquals("endOffset is wrong", expected.endOffset(), docsAndPositionsEnum.endOffset()); + assertEquals("startOffset is wrong", expected.startOffset(), docsEnum.startOffset()); + assertEquals("endOffset is wrong", expected.endOffset(), docsEnum.endOffset()); } else { if (VERBOSE) { System.out.println(" skip check offsets"); @@ -1057,8 +1064,8 @@ if (VERBOSE) { System.out.println(" now check offsets are -1"); } - assertEquals("startOffset isn't -1", -1, docsAndPositionsEnum.startOffset()); - assertEquals("endOffset isn't -1", -1, docsAndPositionsEnum.endOffset()); + assertEquals("startOffset isn't -1", -1, docsEnum.startOffset()); + assertEquals("endOffset isn't -1", -1, docsEnum.endOffset()); } } } @@ -1575,8 +1582,6 @@ if (random().nextBoolean()) { docs = termsEnum.docs(null, docs, DocsEnum.FLAG_FREQS); - } else if (docs instanceof DocsAndPositionsEnum) { - docs = termsEnum.docsAndPositions(null, (DocsAndPositionsEnum) docs, 0); } else { docs = termsEnum.docsAndPositions(null, null, 0); } @@ -1585,12 +1590,9 @@ while (docs.nextDoc() != DocsEnum.NO_MORE_DOCS) { docFreq++; totalTermFreq += docs.freq(); - if (docs instanceof DocsAndPositionsEnum) { - DocsAndPositionsEnum posEnum = (DocsAndPositionsEnum) docs; - int limit = TestUtil.nextInt(random(), 1, docs.freq()); - for(int i=0;i doc : "backwards nextDoc from " + doc + " to " + nextDoc + " " + in; if (nextDoc == DocIdSetIterator.NO_MORE_DOCS) { state = DocsEnumState.FINISHED; + positionMax = 0; } else { state = DocsEnumState.ITERATING; + positionMax = super.freq(); } + positionCount = 0; assert super.docID() == nextDoc; return doc = nextDoc; } @@ -296,9 +301,12 @@ assert advanced >= target : "backwards advance from: " + target + " to: " + advanced; if (advanced == DocIdSetIterator.NO_MORE_DOCS) { state = DocsEnumState.FINISHED; + positionMax = 0; } else { state = DocsEnumState.ITERATING; + positionMax = super.freq(); } + positionCount = 0; assert super.docID() == advanced; return doc = advanced; } @@ -317,15 +325,72 @@ assert freq > 0; return freq; } + + @Override + public int nextPosition() throws IOException { + assert state != DocsEnumState.START : "nextPosition() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "nextPosition() called after NO_MORE_DOCS"; + int position = super.nextPosition(); + assert position >= 0 || position == -1 : "invalid position: " + position; + if (positionCount++ >= positionMax) + assert position == NO_MORE_POSITIONS : "nextPosition() does not return NO_MORE_POSITIONS when exhausted"; + return position; + } + + @Override + public int startOffset() throws IOException { + assert state != DocsEnumState.START : "startOffset() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "startOffset() called after NO_MORE_DOCS"; + assert positionCount > 0 : "startOffset() called before nextPosition()!"; + assert positionCount <= positionMax : "startOffset() called after NO_MORE_POSITIONS"; + return super.startOffset(); + } + + @Override + public int endOffset() throws IOException { + assert state != DocsEnumState.START : "endOffset() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "endOffset() called after NO_MORE_DOCS"; + assert positionCount > 0 : "endOffset() called before nextPosition()!"; + assert positionCount <= positionMax : "endOffset() called after NO_MORE_POSITIONS"; + return super.endOffset(); + } + + @Override + public int startPosition() throws IOException { + assert state != DocsEnumState.START : "startPosition() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "startPosition() called after NO_MORE_DOCS"; + assert positionCount > 0 : "startPosition() called before nextPosition()!"; + assert positionCount <= positionMax : "startPosition() called after NO_MORE_POSITIONS"; + return super.startPosition(); + } + + @Override + public int endPosition() throws IOException { + assert state != DocsEnumState.START : "endPosition() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "endPosition() called after NO_MORE_DOCS"; + assert positionCount > 0 : "endPosition() called before nextPosition()!"; + assert positionCount <= positionMax : "endPosition() called after NO_MORE_POSITIONS"; + return super.endPosition(); + } + + @Override + public BytesRef getPayload() throws IOException { + assert state != DocsEnumState.START : "getPayload() called before nextDoc()/advance()"; + assert state != DocsEnumState.FINISHED : "getPayload() called after NO_MORE_DOCS"; + assert positionCount > 0 : "getPayload() called before nextPosition()!"; + BytesRef payload = super.getPayload(); + assert payload == null || payload.length > 0 : "getPayload() returned payload with invalid length!"; + return payload; + } } - static class AssertingDocsAndPositionsEnum extends FilterDocsAndPositionsEnum { + static class AssertingDocsAndPositionsEnum extends FilterDocsEnum { private DocsEnumState state = DocsEnumState.START; private int positionMax = 0; private int positionCount = 0; private int doc; - public AssertingDocsAndPositionsEnum(DocsAndPositionsEnum in) { + public AssertingDocsAndPositionsEnum(DocsEnum in) { super(in); int docid = in.docID(); assert docid == -1 : "invalid initial doc id: " + docid; Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (.../trunk) (revision 1643331) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (.../branches/lucene2878) (revision 1643331) @@ -26,7 +26,6 @@ import java.util.TreeMap; import org.apache.lucene.codecs.FieldsProducer; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -70,8 +69,8 @@ class SimpleTextFieldsReader extends FieldsProducer { private static final long BASE_RAM_BYTES_USED = - RamUsageEstimator.shallowSizeOfInstance(SimpleTextFieldsReader.class) - + RamUsageEstimator.shallowSizeOfInstance(TreeMap.class); + RamUsageEstimator.shallowSizeOfInstance(SimpleTextFieldsReader.class) + + RamUsageEstimator.shallowSizeOfInstance(TreeMap.class); private final TreeMap fields; private final IndexInput in; @@ -92,12 +91,12 @@ } } } - + private TreeMap readFields(IndexInput in) throws IOException { ChecksumIndexInput input = new BufferedChecksumIndexInput(in); BytesRefBuilder scratch = new BytesRefBuilder(); TreeMap fields = new TreeMap<>(); - + while (true) { SimpleTextUtil.readLine(input, scratch); if (scratch.get().equals(END)) { @@ -205,9 +204,13 @@ public long totalTermFreq() { return indexOptions == IndexOptions.DOCS ? -1 : totalTermFreq; } - + @Override public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { + + if ((flags & DocsEnum.FLAG_POSITIONS) >= DocsEnum.FLAG_POSITIONS) + return docsAndPositions(liveDocs, reuse, flags); + SimpleTextDocsEnum docsEnum; if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) { docsEnum = (SimpleTextDocsEnum) reuse; @@ -218,7 +221,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + public DocsEnum docsAndPositions(Bits liveDocs, DocsEnum reuse, int flags) throws IOException { if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed @@ -230,7 +233,7 @@ docsAndPositionsEnum = (SimpleTextDocsAndPositionsEnum) reuse; } else { docsAndPositionsEnum = new SimpleTextDocsAndPositionsEnum(); - } + } return docsAndPositionsEnum.reset(docsStart, liveDocs, indexOptions, docFreq); } } @@ -245,7 +248,7 @@ private final BytesRefBuilder scratch = new BytesRefBuilder(); private final CharsRefBuilder scratchUTF16 = new CharsRefBuilder(); private int cost; - + public SimpleTextDocsEnum() { this.inStart = SimpleTextFieldsReader.this.in; this.in = this.inStart.clone(); @@ -276,6 +279,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int nextDoc() throws IOException { if (docID == NO_MORE_DOCS) { return docID; @@ -327,7 +360,7 @@ // Naive -- better to index skip data return slowAdvance(target); } - + @Override public long cost() { return cost; @@ -334,7 +367,7 @@ } } - private class SimpleTextDocsAndPositionsEnum extends DocsAndPositionsEnum { + private class SimpleTextDocsAndPositionsEnum extends DocsEnum { private final IndexInput inStart; private final IndexInput in; private int docID = -1; @@ -344,6 +377,7 @@ private final BytesRefBuilder scratch2 = new BytesRefBuilder(); private final CharsRefBuilder scratchUTF16 = new CharsRefBuilder(); private final CharsRefBuilder scratchUTF16_2 = new CharsRefBuilder(); + private int pos; private BytesRef payload; private long nextDocStart; private boolean readOffsets; @@ -436,10 +470,12 @@ @Override public int nextPosition() throws IOException { - final int pos; if (readPositions) { SimpleTextUtil.readLine(in, scratch); - assert StringHelper.startsWith(scratch.get(), POS): "got line=" + scratch.get().utf8ToString(); + if (!StringHelper.startsWith(scratch.get(), POS)) { + pos = NO_MORE_POSITIONS; + return pos; + } scratchUTF16_2.copyUTF8Bytes(scratch.bytes(), POS.length, scratch.length()-POS.length); pos = ArrayUtil.parseInt(scratchUTF16_2.chars(), 0, scratchUTF16_2.length()); } else { @@ -474,6 +510,16 @@ } @Override + public int startPosition() throws IOException { + return pos; + } + + @Override + public int endPosition() throws IOException { + return pos; + } + + @Override public int startOffset() throws IOException { return startOffset; } @@ -487,7 +533,7 @@ public BytesRef getPayload() { return payload; } - + @Override public long cost() { return cost; @@ -505,9 +551,9 @@ } private static final long TERMS_BASE_RAM_BYTES_USED = - RamUsageEstimator.shallowSizeOfInstance(SimpleTextTerms.class) - + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class) - + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class); + RamUsageEstimator.shallowSizeOfInstance(SimpleTextTerms.class) + + RamUsageEstimator.shallowSizeOfInstance(BytesRef.class) + + RamUsageEstimator.shallowSizeOfInstance(CharsRef.class); private class SimpleTextTerms extends Terms implements Accountable { private final long termsStart; private final FieldInfo fieldInfo; @@ -532,7 +578,7 @@ final Builder>> b; final PairOutputs outputsInner = new PairOutputs<>(posIntOutputs, posIntOutputs); final PairOutputs> outputs = new PairOutputs<>(posIntOutputs, - outputsInner); + outputsInner); b = new Builder<>(FST.INPUT_TYPE.BYTE1, outputs); IndexInput in = SimpleTextFieldsReader.this.in.clone(); in.seek(termsStart); @@ -547,8 +593,8 @@ if (scratch.get().equals(END) || StringHelper.startsWith(scratch.get(), FIELD)) { if (lastDocsStart != -1) { b.add(Util.toIntsRef(lastTerm.get(), scratchIntsRef), - outputs.newPair(lastDocsStart, - outputsInner.newPair((long) docFreq, totalTermFreq))); + outputs.newPair(lastDocsStart, + outputsInner.newPair((long) docFreq, totalTermFreq))); sumTotalTermFreq += totalTermFreq; } break; @@ -564,7 +610,7 @@ } else if (StringHelper.startsWith(scratch.get(), TERM)) { if (lastDocsStart != -1) { b.add(Util.toIntsRef(lastTerm.get(), scratchIntsRef), outputs.newPair(lastDocsStart, - outputsInner.newPair((long) docFreq, totalTermFreq))); + outputsInner.newPair((long) docFreq, totalTermFreq))); } lastDocsStart = in.getFilePointer(); final int len = scratch.length() - TERM.length; @@ -651,7 +697,7 @@ public boolean hasPositions() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; } - + @Override public boolean hasPayloads() { return fieldInfo.hasPayloads(); Index: lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/search/AssertingIndexSearcher.java (.../branches/lucene2878) (revision 1643331) @@ -17,16 +17,17 @@ * limitations under the License. */ +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.TestUtil; + import java.io.IOException; import java.util.List; import java.util.Random; import java.util.concurrent.ExecutorService; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReaderContext; -import org.apache.lucene.util.TestUtil; - /** * Helper class that adds some extra checks to ensure correct * usage of {@code IndexSearcher} and {@code Weight}. @@ -65,6 +66,21 @@ } @Override + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + Scorer scorer = w.scorer(context, flags, acceptDocs); + if (scorer != null) { + // check that scorer obeys disi contract for docID() before next()/advance + try { + int docid = scorer.docID(); + assert docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS; + } catch (UnsupportedOperationException ignored) { + // from a top-level BS1 + } + } + return scorer; + } + + @Override public float getValueForNormalization() { throw new IllegalStateException("Weight already normalized."); } Index: lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/search/AssertingScorer.java (.../branches/lucene2878) (revision 1643331) @@ -26,6 +26,7 @@ import java.util.WeakHashMap; import org.apache.lucene.index.AssertingLeafReader; +import org.apache.lucene.util.BytesRef; /** Wraps a Scorer with additional checks */ public class AssertingScorer extends Scorer { @@ -111,6 +112,42 @@ } @Override + public int nextPosition() throws IOException { + assert iterating(); + return in.nextPosition(); + } + + @Override + public int startPosition() throws IOException { + assert iterating(); + return in.startPosition(); + } + + @Override + public int endPosition() throws IOException { + assert iterating(); + return in.endPosition(); + } + + @Override + public int startOffset() throws IOException { + assert iterating(); + return in.startOffset(); + } + + @Override + public int endOffset() throws IOException { + assert iterating(); + return in.endOffset(); + } + + @Override + public BytesRef getPayload() throws IOException { + assert iterating(); + return in.getPayload(); + } + + @Override public int docID() { return in.docID(); } @@ -117,12 +154,16 @@ @Override public int nextDoc() throws IOException { - return docsEnumIn.nextDoc(); + int doc = docsEnumIn.nextDoc(); + assert in.startPosition() == -1; + return doc; } @Override public int advance(int target) throws IOException { - return docsEnumIn.advance(target); + int doc = docsEnumIn.advance(target); + assert in.startPosition() == -1; + return doc; } @Override Index: lucene/codecs =================================================================== --- lucene/codecs (.../trunk) (revision 1643331) +++ lucene/codecs (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/codecs ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/codecs:r1638800-1643324 Index: solr/licenses =================================================================== --- solr/licenses (.../trunk) (revision 1643331) +++ solr/licenses (.../branches/lucene2878) (revision 1643331) Property changes on: solr/licenses ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/licenses:r1638800-1642144 Index: solr/bin =================================================================== --- solr/bin (.../trunk) (revision 1643331) +++ solr/bin (.../branches/lucene2878) (revision 1643331) Property changes on: solr/bin ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/bin:r1638800-1643324 Index: solr/test-framework =================================================================== --- solr/test-framework (.../trunk) (revision 1643331) +++ solr/test-framework (.../branches/lucene2878) (revision 1643331) Property changes on: solr/test-framework ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/test-framework:r1638800-1642144 Index: solr/README.txt =================================================================== --- solr/README.txt (.../trunk) (revision 1643331) +++ solr/README.txt (.../branches/lucene2878) (revision 1643331) Property changes on: solr/README.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/README.txt:r1638800-1642144 Index: solr/webapp =================================================================== --- solr/webapp (.../trunk) (revision 1643331) +++ solr/webapp (.../branches/lucene2878) (revision 1643331) Property changes on: solr/webapp ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/webapp:r1638800-1643324 Index: lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (.../branches/lucene2878) (revision 1643331) @@ -43,6 +43,8 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import com.carrotsearch.randomizedtesting.generators.RandomInts; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; @@ -67,7 +69,6 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FilterLeafReader; import org.apache.lucene.index.IndexReader; @@ -86,8 +87,8 @@ import org.apache.lucene.mockfile.FilterFileSystem; import org.apache.lucene.mockfile.WindowsFS; import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.FilteredQuery.FilterStrategy; -import org.apache.lucene.search.FilteredQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; @@ -96,8 +97,6 @@ import org.apache.lucene.store.NoLockFactory; import org.junit.Assert; -import com.carrotsearch.randomizedtesting.generators.RandomInts; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; /** * General utility methods for Lucene unit tests. @@ -959,13 +958,13 @@ if (random.nextBoolean()) { final int posFlags; switch (random.nextInt(4)) { - case 0: posFlags = 0; break; - case 1: posFlags = DocsAndPositionsEnum.FLAG_OFFSETS; break; - case 2: posFlags = DocsAndPositionsEnum.FLAG_PAYLOADS; break; - default: posFlags = DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS; break; + case 0: posFlags = DocsEnum.FLAG_POSITIONS; break; + case 1: posFlags = DocsEnum.FLAG_OFFSETS; break; + case 2: posFlags = DocsEnum.FLAG_PAYLOADS; break; + default: posFlags = DocsEnum.FLAG_OFFSETS | DocsEnum.FLAG_PAYLOADS; break; } // TODO: cast to DocsAndPositionsEnum? - DocsAndPositionsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null, posFlags); + DocsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null, posFlags); if (docsAndPositions != null) { return docsAndPositions; } Index: lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (.../trunk) (revision 1643331) +++ lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java (.../branches/lucene2878) (revision 1643331) @@ -55,59 +55,42 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; +import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; +import com.carrotsearch.randomizedtesting.LifecycleScope; +import com.carrotsearch.randomizedtesting.MixWithSuiteName; +import com.carrotsearch.randomizedtesting.RandomizedContext; +import com.carrotsearch.randomizedtesting.RandomizedRunner; +import com.carrotsearch.randomizedtesting.RandomizedTest; +import com.carrotsearch.randomizedtesting.annotations.Listeners; +import com.carrotsearch.randomizedtesting.annotations.SeedDecorators; +import com.carrotsearch.randomizedtesting.annotations.TestGroup; +import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup.Group; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies; +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule; +import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule; +import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule; +import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.AlcoholicMergePolicy; -import org.apache.lucene.index.AssertingDirectoryReader; -import org.apache.lucene.index.AssertingLeafReader; -import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.index.CompositeReader; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldFilterLeafReader; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Fields; -import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.*; import org.apache.lucene.index.IndexReader.ReaderClosedListener; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.LiveIndexWriterConfig; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.MergePolicy; -import org.apache.lucene.index.MergeScheduler; -import org.apache.lucene.index.MockRandomMergePolicy; -import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.NumericDocValues; -import org.apache.lucene.index.ParallelCompositeReader; -import org.apache.lucene.index.ParallelLeafReader; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.index.SerialMergeScheduler; -import org.apache.lucene.index.SimpleMergedSegmentWarmer; -import org.apache.lucene.index.SlowCompositeReaderWrapper; -import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.index.SortedSetDocValues; -import org.apache.lucene.index.StorableField; -import org.apache.lucene.index.StoredDocument; -import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.DocIdSetIterator; @@ -120,12 +103,12 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FSLockFactory; import org.apache.lucene.store.FlushInfo; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IOContext.Context; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.LockFactory; import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.MockDirectoryWrapper.Throttling; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.NRTCachingDirectory; import org.apache.lucene.store.RateLimitedDirectoryWrapper; import org.apache.lucene.util.automaton.AutomatonTestUtil; @@ -143,33 +126,6 @@ import org.junit.rules.TestRule; import org.junit.runner.RunWith; -import com.carrotsearch.randomizedtesting.JUnit4MethodProvider; -import com.carrotsearch.randomizedtesting.LifecycleScope; -import com.carrotsearch.randomizedtesting.MixWithSuiteName; -import com.carrotsearch.randomizedtesting.RandomizedContext; -import com.carrotsearch.randomizedtesting.RandomizedRunner; -import com.carrotsearch.randomizedtesting.RandomizedTest; -import com.carrotsearch.randomizedtesting.annotations.Listeners; -import com.carrotsearch.randomizedtesting.annotations.SeedDecorators; -import com.carrotsearch.randomizedtesting.annotations.TestGroup; -import com.carrotsearch.randomizedtesting.annotations.TestMethodProviders; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction.Action; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakAction; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup.Group; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakGroup; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope.Scope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies.Consequence; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakZombies; -import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; -import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import com.carrotsearch.randomizedtesting.rules.NoClassHooksShadowingRule; -import com.carrotsearch.randomizedtesting.rules.NoInstanceHooksOverridesRule; -import com.carrotsearch.randomizedtesting.rules.StaticFieldsInvariantRule; -import com.carrotsearch.randomizedtesting.rules.SystemPropertiesInvariantRule; - import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsBoolean; import static com.carrotsearch.randomizedtesting.RandomizedTest.systemPropertyAsInt; @@ -468,9 +424,9 @@ static { CORE_DIRECTORIES = new ArrayList<>(FS_DIRECTORIES); CORE_DIRECTORIES.add("RAMDirectory"); - }; + } - /** A {@link FilterCachingPolicy} that randomly caches. */ + /** A {@link org.apache.lucene.search.FilterCachingPolicy} that randomly caches. */ public static final FilterCachingPolicy MAYBE_CACHE_POLICY = new FilterCachingPolicy() { @Override @@ -482,7 +438,7 @@ } }; - + // ----------------------------------------------------------------- // Fields initialized in class or instance rules. // ----------------------------------------------------------------- @@ -1291,10 +1247,6 @@ String fsdirClass = TEST_DIRECTORY; if (fsdirClass.equals("random")) { fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); - if (fsdirClass.equals("SimpleFSDirectory")) { - // pick again - fsdirClass = RandomPicks.randomFrom(random(), FS_DIRECTORIES); - } } Class clazz; @@ -1334,7 +1286,7 @@ directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble()); } - if (TEST_NIGHTLY && rarely(random) && !bare) { + if (rarely(random) && !bare) { final double maxMBPerSec = TestUtil.nextInt(random, 20, 40); if (LuceneTestCase.VERBOSE) { System.out.println("LuceneTestCase: will rate limit output IndexOutput to " + maxMBPerSec + " MB/sec"); @@ -1475,7 +1427,7 @@ /** * Return a random Locale from the available locales on the system. - * @see LUCENE-4020 + * @see "https://issues.apache.org/jira/browse/LUCENE-4020" */ public static Locale randomLocale(Random random) { Locale locales[] = Locale.getAvailableLocales(); @@ -1484,7 +1436,7 @@ /** * Return a random TimeZone from the available timezones on the system - * @see LUCENE-4020 + * @see "https://issues.apache.org/jira/browse/LUCENE-4020" */ public static TimeZone randomTimeZone(Random random) { String tzIds[] = TimeZone.getAvailableIDs(); @@ -1521,10 +1473,6 @@ if (clazzName.equals("random")) { if (rarely(random)) { clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES); - if (clazzName.equals("SimpleFSDirectory")) { - // pick again - clazzName = RandomPicks.randomFrom(random, CORE_DIRECTORIES); - } } else { clazzName = "RAMDirectory"; } @@ -1930,8 +1878,8 @@ public void assertTermsEnumEquals(String info, IndexReader leftReader, TermsEnum leftTermsEnum, TermsEnum rightTermsEnum, boolean deep) throws IOException { BytesRef term; Bits randomBits = new RandomBits(leftReader.maxDoc(), random().nextDouble(), random()); - DocsAndPositionsEnum leftPositions = null; - DocsAndPositionsEnum rightPositions = null; + DocsEnum leftPositions = null; + DocsEnum rightPositions = null; DocsEnum leftDocs = null; DocsEnum rightDocs = null; @@ -1995,7 +1943,7 @@ /** * checks docs + freqs + positions + payloads, sequentially */ - public void assertDocsAndPositionsEnumEquals(String info, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws IOException { + public void assertDocsAndPositionsEnumEquals(String info, DocsEnum leftDocs, DocsEnum rightDocs) throws IOException { if (leftDocs == null || rightDocs == null) { assertNull(leftDocs); assertNull(rightDocs); @@ -2074,7 +2022,7 @@ /** * checks advancing docs + positions */ - public void assertPositionsSkippingEquals(String info, IndexReader leftReader, int docFreq, DocsAndPositionsEnum leftDocs, DocsAndPositionsEnum rightDocs) throws IOException { + public void assertPositionsSkippingEquals(String info, IndexReader leftReader, int docFreq, DocsEnum leftDocs, DocsEnum rightDocs) throws IOException { if (leftDocs == null || rightDocs == null) { assertNull(leftDocs); assertNull(rightDocs); Index: dev-tools/idea/lucene/highlighter/highlighter.iml =================================================================== --- dev-tools/idea/lucene/highlighter/highlighter.iml (.../trunk) (revision 1643331) +++ dev-tools/idea/lucene/highlighter/highlighter.iml (.../branches/lucene2878) (revision 1643331) @@ -12,6 +12,7 @@ + Index: dev-tools =================================================================== --- dev-tools (.../trunk) (revision 1643331) +++ dev-tools (.../branches/lucene2878) (revision 1643331) Property changes on: dev-tools ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/dev-tools:r1638800-1642714 Index: lucene/test-framework =================================================================== --- lucene/test-framework (.../trunk) (revision 1643331) +++ lucene/test-framework (.../branches/lucene2878) (revision 1643331) Property changes on: lucene/test-framework ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene/test-framework:r1638800-1643324 Index: lucene =================================================================== --- lucene (.../trunk) (revision 1643331) +++ lucene (.../branches/lucene2878) (revision 1643331) Property changes on: lucene ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene:r1638800-1643324 Index: solr/cloud-dev =================================================================== --- solr/cloud-dev (.../trunk) (revision 1643331) +++ solr/cloud-dev (.../branches/lucene2878) (revision 1643331) Property changes on: solr/cloud-dev ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/cloud-dev:r1638800-1643324 Index: solr/common-build.xml =================================================================== --- solr/common-build.xml (.../trunk) (revision 1643331) +++ solr/common-build.xml (.../branches/lucene2878) (revision 1643331) Property changes on: solr/common-build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/common-build.xml:r1638800-1642714 Index: solr/CHANGES.txt =================================================================== --- solr/CHANGES.txt (.../trunk) (revision 1643331) +++ solr/CHANGES.txt (.../branches/lucene2878) (revision 1643331) Property changes on: solr/CHANGES.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/CHANGES.txt:r1638800-1643324 Index: solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java =================================================================== --- solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (.../branches/lucene2878) (revision 1643331) @@ -1,15 +1,24 @@ package org.apache.solr.search; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.search.*; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.ConstantScoreQuery; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.Filter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.Bits; -import org.apache.lucene.index.IndexReader; import org.apache.solr.common.SolrException; import java.io.IOException; +import java.util.Map; import java.util.Set; -import java.util.Map; /* * Licensed to the Apache Software Foundation (ASF) under one or more @@ -119,7 +128,7 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new ConstantScorer(context, this, queryWeight, acceptDocs); } @@ -192,6 +201,11 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override public int advance(int target) throws IOException { return docIdSetIterator.advance(target); } Index: solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,10 @@ package org.apache.solr.search; -import java.io.IOException; -import java.util.Arrays; -import java.util.Iterator; -import java.util.Map; - +import com.carrotsearch.hppc.FloatArrayList; +import com.carrotsearch.hppc.IntIntOpenHashMap; +import com.carrotsearch.hppc.IntOpenHashSet; +import com.carrotsearch.hppc.cursors.IntIntCursor; import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -49,10 +48,10 @@ import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; -import com.carrotsearch.hppc.FloatArrayList; -import com.carrotsearch.hppc.IntIntOpenHashMap; -import com.carrotsearch.hppc.IntOpenHashSet; -import com.carrotsearch.hppc.cursors.IntIntCursor; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; /** @@ -354,6 +353,11 @@ return 0; } + @Override + public int nextPosition() throws IOException { + return -1; + } + public int advance(int i) { return -1; } Index: solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/ReRankQParserPlugin.java (.../branches/lucene2878) (revision 1643331) @@ -194,8 +194,8 @@ return mainWeight.getValueForNormalization(); } - public Scorer scorer(LeafReaderContext context, Bits bits) throws IOException { - return mainWeight.scorer(context, bits); + public Scorer scorer(LeafReaderContext context, int flags, Bits bits) throws IOException { + return mainWeight.scorer(context, flags, bits); } public Query getQuery() { Index: solr/core/src/java/org/apache/solr/schema/LatLonType.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/LatLonType.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/schema/LatLonType.java (.../branches/lucene2878) (revision 1643331) @@ -16,15 +16,12 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Set; - +import com.spatial4j.core.context.SpatialContext; +import com.spatial4j.core.distance.DistanceUtils; import com.spatial4j.core.shape.Point; - +import com.spatial4j.core.shape.Rectangle; import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.StorableField; @@ -49,14 +46,15 @@ import org.apache.solr.search.PostFilter; import org.apache.solr.search.QParser; import org.apache.solr.search.SpatialOptions; +import org.apache.solr.util.SpatialUtils; -import com.spatial4j.core.context.SpatialContext; -import com.spatial4j.core.distance.DistanceUtils; -import com.spatial4j.core.shape.Rectangle; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; -import org.apache.solr.util.SpatialUtils; - /** * Represents a Latitude/Longitude as a 2 dimensional point. Latitude is always specified first. */ @@ -341,13 +339,13 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { return new SpatialScorer(context, acceptDocs, this, queryWeight); } @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - return ((SpatialScorer)scorer(context, context.reader().getLiveDocs())).explain(doc); + return ((SpatialScorer)scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs())).explain(doc); } } @@ -484,6 +482,11 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override public long cost() { return maxDoc; } @@ -506,6 +509,7 @@ result.addDetail(new Explanation(weight.queryNorm,"queryNorm")); return result; } + } @Override Index: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (.../branches/lucene2878) (revision 1643331) @@ -16,17 +16,10 @@ */ package org.apache.solr.search; -import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; - -import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocsEnum; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -57,6 +50,12 @@ import org.apache.solr.schema.TrieField; import org.apache.solr.util.RefCounted; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Set; public class JoinQParserPlugin extends QParserPlugin { public static final String NAME = "join"; @@ -232,7 +231,7 @@ @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { if (filter == null) { boolean debug = rb != null && rb.isDebug(); long start = debug ? System.currentTimeMillis() : 0; @@ -501,7 +500,7 @@ @Override public Explanation explain(LeafReaderContext context, int doc) throws IOException { - Scorer scorer = scorer(context, context.reader().getLiveDocs()); + Scorer scorer = scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); boolean exists = scorer.advance(doc) == doc; ComplexExplanation result = new ComplexExplanation(); @@ -556,6 +555,11 @@ } @Override + public int nextPosition() throws IOException { + return -1; + } + + @Override public int advance(int target) throws IOException { return iter.advance(target); } Index: solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java =================================================================== --- solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (.../branches/lucene2878) (revision 1643331) @@ -17,23 +17,6 @@ package org.apache.solr.search; -import java.io.Closeable; -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; @@ -61,34 +44,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.LeafCollector; -import org.apache.lucene.search.BooleanClause; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.Collector; -import org.apache.lucene.search.ConstantScoreQuery; -import org.apache.lucene.search.DocIdSet; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.Filter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.search.MultiCollector; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.SimpleCollector; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TimeLimitingCollector; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TopDocsCollector; -import org.apache.lucene.search.TopFieldCollector; -import org.apache.lucene.search.TopFieldDocs; -import org.apache.lucene.search.TopScoreDocCollector; -import org.apache.lucene.search.TotalHitCountCollector; -import org.apache.lucene.search.Weight; +import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.uninverting.UninvertingReader; import org.apache.lucene.util.Bits; @@ -115,7 +71,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Closeable; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + /** * SolrIndexSearcher adds schema awareness and caching functionality * over the lucene IndexSearcher. @@ -2461,7 +2434,7 @@ iterators.add(iter); } for (Weight w : weights) { - Scorer scorer = w.scorer(context, context.reader().getLiveDocs()); + Scorer scorer = w.scorer(context, DocsEnum.FLAG_FREQS, context.reader().getLiveDocs()); if (scorer == null) return null; iterators.add(scorer); } Index: solr/solrj =================================================================== --- solr/solrj (.../trunk) (revision 1643331) +++ solr/solrj (.../branches/lucene2878) (revision 1643331) Property changes on: solr/solrj ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/solrj:r1638800-1643324 Index: solr/example =================================================================== --- solr/example (.../trunk) (revision 1643331) +++ solr/example (.../branches/lucene2878) (revision 1643331) Property changes on: solr/example ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/example:r1638800-1642144 Index: solr/build.xml =================================================================== --- solr/build.xml (.../trunk) (revision 1643331) +++ solr/build.xml (.../branches/lucene2878) (revision 1643331) Property changes on: solr/build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/build.xml:r1638800-1643324 Index: solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java =================================================================== --- solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/search/join/IgnoreAcceptDocsQuery.java (.../branches/lucene2878) (revision 1643331) @@ -86,8 +86,8 @@ } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { - return w.scorer(context, null); + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + return w.scorer(context, flags, null); } } Index: solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java (.../branches/lucene2878) (revision 1643331) @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -31,8 +32,8 @@ import java.util.Locale; import java.util.Map; +import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanQuery; @@ -52,7 +53,13 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; -import org.apache.solr.common.params.*; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.CursorMarkParams; +import org.apache.solr.common.params.GroupParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.MoreLikeThisParams; +import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.StrUtils; @@ -71,11 +78,11 @@ import org.apache.solr.search.QParser; import org.apache.solr.search.QParserPlugin; import org.apache.solr.search.QueryParsing; +import org.apache.solr.search.RankQuery; import org.apache.solr.search.ReturnFields; import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.search.SolrReturnFields; import org.apache.solr.search.SortSpec; -import org.apache.solr.search.RankQuery; import org.apache.solr.search.SyntaxError; import org.apache.solr.search.grouping.CommandHandler; import org.apache.solr.search.grouping.GroupingSpecification; @@ -97,7 +104,6 @@ import org.apache.solr.search.grouping.endresulttransformer.MainEndResultTransformer; import org.apache.solr.search.grouping.endresulttransformer.SimpleEndResultTransformer; import org.apache.solr.util.SolrPluginUtils; -import java.util.Collections; /** * TODO! @@ -1298,6 +1304,36 @@ } @Override + public int nextPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endPosition() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int startOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int endOffset() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef getPayload() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override public int nextDoc() throws IOException { throw new UnsupportedOperationException(); } Index: solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/handler/component/TermVectorComponent.java (.../branches/lucene2878) (revision 1643331) @@ -13,7 +13,7 @@ import java.util.Map; import java.util.Map.Entry; -import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; @@ -335,7 +335,7 @@ docNL.add(field, fieldNL); BytesRef text; - DocsAndPositionsEnum dpEnum = null; + DocsEnum dpEnum = null; while((text = termsEnum.next()) != null) { String term = text.utf8ToString(); NamedList termInfo = new NamedList<>(); Index: solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java (.../branches/lucene2878) (revision 1643331) @@ -17,15 +17,13 @@ package org.apache.solr.handler.component; -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - +import com.carrotsearch.hppc.IntObjectMap; +import com.carrotsearch.hppc.IntObjectOpenHashMap; +import com.carrotsearch.hppc.IntOpenHashSet; +import com.carrotsearch.hppc.cursors.IntObjectCursor; +import com.carrotsearch.hppc.cursors.ObjectCursor; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedDocValues; @@ -63,11 +61,13 @@ import org.apache.solr.util.plugin.PluginInfoInitialized; import org.apache.solr.util.plugin.SolrCoreAware; -import com.carrotsearch.hppc.IntObjectMap; -import com.carrotsearch.hppc.IntObjectOpenHashMap; -import com.carrotsearch.hppc.IntOpenHashSet; -import com.carrotsearch.hppc.cursors.IntObjectCursor; -import com.carrotsearch.hppc.cursors.ObjectCursor; +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * The ExpandComponent is designed to work with the CollapsingPostFilter. @@ -352,6 +352,11 @@ }; } + @Override + public int postingFeatures() { + return DocsEnum.FLAG_NONE; + } + public IntObjectMap getGroups() { return groups; } Index: solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java =================================================================== --- solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java (.../trunk) (revision 1643331) +++ solr/core/src/java/org/apache/solr/update/DeleteByQueryWrapper.java (.../branches/lucene2878) (revision 1643331) @@ -17,11 +17,9 @@ * limitations under the License. */ -import java.io.IOException; - +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -31,6 +29,8 @@ import org.apache.lucene.util.Bits; import org.apache.solr.schema.IndexSchema; +import java.io.IOException; + /** * Allows access to uninverted docvalues by delete-by-queries. * this is used e.g. to implement versioning constraints in solr. @@ -82,8 +82,8 @@ public void normalize(float norm, float topLevelBoost) { inner.normalize(norm, topLevelBoost); } @Override - public Scorer scorer(LeafReaderContext context, Bits acceptDocs) throws IOException { - return inner.scorer(privateContext.getIndexReader().leaves().get(0), acceptDocs); + public Scorer scorer(LeafReaderContext context, int flags, Bits acceptDocs) throws IOException { + return inner.scorer(privateContext.getIndexReader().leaves().get(0), flags, acceptDocs); } }; } Index: solr/core =================================================================== --- solr/core (.../trunk) (revision 1643331) +++ solr/core (.../branches/lucene2878) (revision 1643331) Property changes on: solr/core ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/core:r1638800-1643324 Index: solr/contrib =================================================================== --- solr/contrib (.../trunk) (revision 1643331) +++ solr/contrib (.../branches/lucene2878) (revision 1643331) Property changes on: solr/contrib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr/contrib:r1638800-1643324 Index: solr =================================================================== --- solr (.../trunk) (revision 1643331) +++ solr (.../branches/lucene2878) (revision 1643331) Property changes on: solr ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr:r1638800-1643324 Index: . =================================================================== --- . (.../trunk) (revision 1643331) +++ . (.../branches/lucene2878) (revision 1643331) Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk:r1638800-1643324