diff --git a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java index b315edf..16aa231 100644 --- a/lucene/core/src/java/org/apache/lucene/search/TermScorer.java +++ b/lucene/core/src/java/org/apache/lucene/search/TermScorer.java @@ -118,6 +118,8 @@ final class TermScorer extends Scorer { public PositionInterval next() throws IOException { if (--positionsPending >= 0) { interval.begin = interval.end = docsAndPos.nextPosition(); + interval.offsetBegin = docsAndPos.startOffset(); + interval.offsetEnd = docsAndPos.endOffset(); return interval; } positionsPending = 0; diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java b/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java index 6a985a7..9f2c1d2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/BlockPositionIterator.java @@ -30,10 +30,10 @@ public final class BlockPositionIterator extends PositionIntervalIterator { private final PositionIntervalIterator[] iterators; private static final PositionInterval INFINITE_INTERVAL = new PositionInterval( - Integer.MIN_VALUE, Integer.MIN_VALUE); + Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); private final PositionInterval[] intervals; private final PositionInterval interval = new PositionInterval( - Integer.MIN_VALUE, Integer.MIN_VALUE); + Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); private final int[] gaps; private final int lastIter; @@ -111,6 +111,8 @@ public final class BlockPositionIterator extends PositionIntervalIterator { } interval.begin = intervals[0].begin; interval.end = intervals[lastIter].end; + interval.offsetBegin = intervals[0].offsetBegin; + interval.offsetEnd = intervals[lastIter].offsetEnd; return interval; } diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java index 6018713..6f90a49 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueue.java @@ -27,7 +27,7 @@ import org.apache.lucene.util.PriorityQueue; // nocommit - javadoc abstract class IntervalQueue extends PriorityQueue { final PositionInterval queueInterval = new PositionInterval( - Integer.MIN_VALUE, Integer.MIN_VALUE); + Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); public void reset() { clear(); diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java index dd2b5ef..366e798 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueAnd.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.positions.PositionIntervalIterator.PositionInter final class IntervalQueueAnd extends IntervalQueue { int rightExtreme = Integer.MIN_VALUE; + int rightExtremeOffset = Integer.MIN_VALUE; public IntervalQueueAnd(int size) { super(size); @@ -34,10 +35,12 @@ final class IntervalQueueAnd extends IntervalQueue { queueInterval.begin = Integer.MIN_VALUE; queueInterval.end = Integer.MIN_VALUE; rightExtreme = Integer.MIN_VALUE; + rightExtremeOffset = Integer.MIN_VALUE; } public void updateRightExtreme(PositionInterval interval) { - rightExtreme = Math.max(rightExtreme, Math.max(interval.end, interval.end)); + rightExtreme = Math.max(rightExtreme, interval.end); + rightExtremeOffset = Math.max(rightExtremeOffset, interval.offsetEnd); } public boolean topContainsQueueInterval() { @@ -49,7 +52,9 @@ final class IntervalQueueAnd extends IntervalQueue { public void updateQueueInterval() { PositionInterval interval = top().interval; queueInterval.begin = interval.begin; + queueInterval.offsetBegin = interval.offsetBegin; queueInterval.end = rightExtreme; + queueInterval.offsetEnd = rightExtremeOffset; } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java index 6cb6d0f..a5a29f2 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/IntervalQueueOr.java @@ -39,7 +39,9 @@ final class IntervalQueueOr extends IntervalQueue { public void updateQueueInterval() { PositionInterval interval = top().interval; queueInterval.begin = interval.begin; + queueInterval.offsetBegin = interval.offsetBegin; queueInterval.end = interval.end; + queueInterval.offsetEnd = interval.offsetEnd; } @Override diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java b/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java index 7360221..84bd2c3 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/OrderedConjunctionPositionIterator.java @@ -26,11 +26,11 @@ public final class OrderedConjunctionPositionIterator extends private final PositionIntervalIterator[] iterators; private static final PositionInterval INFINITE_INTERVAL = new PositionInterval( - Integer.MIN_VALUE, Integer.MIN_VALUE); + Integer.MIN_VALUE, Integer.MIN_VALUE, -1, -1); private final PositionInterval[] intervals; private final int lastIter; private final PositionInterval interval = new PositionInterval( - Integer.MAX_VALUE, Integer.MAX_VALUE); + Integer.MAX_VALUE, Integer.MAX_VALUE, -1, -1); private int index = 1; public OrderedConjunctionPositionIterator(PositionIntervalIterator other) { @@ -52,6 +52,8 @@ public final class OrderedConjunctionPositionIterator extends interval.begin = Integer.MAX_VALUE; interval.end = Integer.MAX_VALUE; + interval.offsetBegin = -1; + interval.offsetEnd = -1; int b = Integer.MAX_VALUE; while (true) { while (true) { @@ -74,6 +76,8 @@ public final class OrderedConjunctionPositionIterator extends } interval.begin = intervals[0].begin; interval.end = intervals[lastIter].end; + interval.offsetBegin = intervals[0].offsetBegin; + interval.offsetEnd = intervals[lastIter].offsetEnd; b = intervals[lastIter].begin; index = 1; intervals[0] = iterators[0].next(); diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java b/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java index 520ce7e..6fcb261 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/PositionFilterQuery.java @@ -50,7 +50,7 @@ public class PositionFilterQuery extends Query implements Cloneable { public Query rewrite(IndexReader reader) throws IOException { PositionFilterQuery clone = null; - Query rewritten = (Query) inner.rewrite(reader); + Query rewritten = inner.rewrite(reader); if (rewritten != inner) { clone = (PositionFilterQuery) this.clone(); clone.inner = rewritten; diff --git a/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java b/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java index d2a502f..8687696 100644 --- a/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java +++ b/lucene/core/src/java/org/apache/lucene/search/positions/PositionIntervalIterator.java @@ -16,11 +16,11 @@ package org.apache.lucene.search.positions; * See the License for the specific language governing permissions and * limitations under the License. */ -import java.io.IOException; - import org.apache.lucene.search.Scorer; import org.apache.lucene.util.BytesRef; +import java.io.IOException; + /** * * @lucene.experimental @@ -89,14 +89,18 @@ public abstract class PositionIntervalIterator { public int begin; public int end; + public int offsetBegin; + public int offsetEnd; - public PositionInterval(int begin, int end) { + public PositionInterval(int begin, int end, int offsetBegin, int offsetEnd) { this.begin = begin; this.end = end; + this.offsetBegin = offsetBegin; + this.offsetEnd = offsetEnd; } public PositionInterval() { - this(0, 0); + this(0, 0, -1, -1); } public boolean nextPayload(BytesRef ref) throws IOException { @@ -122,7 +126,7 @@ public abstract class PositionIntervalIterator { @Override public String toString() { - return "PositionInterval [begin=" + begin + ", end=" + end + "]"; + return "PositionInterval [begin=" + begin + "(" + offsetBegin + "), end=" + end + "(" + offsetEnd + ")]"; } } diff --git a/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java b/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java index 68b86ca..a9ed375 100644 --- a/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/spans/MockSpanQuery.java @@ -58,7 +58,7 @@ public class MockSpanQuery extends SpanQuery { AtomicReaderContext topReaderContext = context.reader().getTopReaderContext(); Weight weight = other.createWeight(new IndexSearcher(topReaderContext)); - Scorer scorer = weight.scorer((AtomicReaderContext) topReaderContext, true, false, acceptDocs); + Scorer scorer = weight.scorer(topReaderContext, true, false, acceptDocs); if (scorer == null) { return EMPTY_SPANS; } diff --git a/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java b/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java index 02e4e14..7d7545b 100644 --- a/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java +++ b/lucene/core/src/test/org/apache/lucene/search/positions/TestPositionOffsets.java @@ -1,4 +1,22 @@ package org.apache.lucene.search.positions; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.codecs.Codec; @@ -9,34 +27,24 @@ import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.*; -import org.apache.lucene.search.*; -import org.apache.lucene.search.positions.PositionIntervalIterator.PositionCollector; -import org.apache.lucene.search.positions.PositionIntervalIterator.PositionInterval; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReaderContext; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.junit.Ignore; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -/** - * Copyright (c) 2012 Lemur Consulting Ltd. - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ public class TestPositionOffsets extends LuceneTestCase { // What am I testing here? diff --git a/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java b/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java index 2fa3fb3..4f95c25 100644 --- a/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java +++ b/lucene/highlighter/src/java/org/apache/lucene/search/poshighlight/PosTokenStream.java @@ -49,12 +49,10 @@ public class PosTokenStream extends TokenStream { // the index of the current position interval private PositionInterval pos = null; - private final PositionOffsetMapper pom; - public PosTokenStream (String text, PositionIntervalIterator positions, PositionOffsetMapper pom) { + public PosTokenStream (String text, PositionIntervalIterator positions) { this.text = text; this.positions = positions; - this.pom = pom; } @Override @@ -64,8 +62,9 @@ public class PosTokenStream extends TokenStream { return false; } int b, e; - b = pom.getStartOffset(pos.begin); - e = pom.getEndOffset(pos.end); + b = pos.offsetBegin; + e = pos.offsetEnd; + assert b >=0; termAtt.append(text, b, e); offsetAtt.setOffset(b, e); posIncrAtt.setPositionIncrement(1); diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java index f7c798b..1601ad3 100644 --- a/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java +++ b/lucene/highlighter/src/test/org/apache/lucene/search/poshighlight/PosHighlighterTest.java @@ -6,12 +6,16 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.codecs.memory.MemoryPostingsFormat; +import org.apache.lucene.codecs.nestedpulsing.NestedPulsingPostingsFormat; +import org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Field.Index; -import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexWriterConfig.OpenMode; @@ -29,266 +33,314 @@ import org.apache.lucene.search.highlight.Highlighter; import org.apache.lucene.search.highlight.InvalidTokenOffsetsException; import org.apache.lucene.search.highlight.SimpleFragmenter; import org.apache.lucene.search.highlight.TextFragment; +import org.apache.lucene.search.positions.BlockPositionIterator; import org.apache.lucene.search.positions.PositionFilterQuery; -import org.apache.lucene.search.positions.TestBlockPositionsIterator.BlockPositionIteratorFilter; +import org.apache.lucene.search.positions.PositionIntervalIterator; +import org.apache.lucene.search.positions.PositionIntervalIterator.PositionIntervalFilter; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.junit.Ignore; /** - * TODO: FIX THIS TEST - * Phrase and Span Queries - * positions callback API + * TODO: FIX THIS TEST Phrase and Span Queries positions callback API */ public class PosHighlighterTest extends LuceneTestCase { - protected final static String F="f"; + protected final static String F = "f"; protected Analyzer analyzer; protected Directory dir; protected IndexSearcher searcher; + private IndexWriterConfig iwc; - private static final String PORRIDGE_VERSE = - "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some" - + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!"; + private static final String PORRIDGE_VERSE = "Pease porridge hot! Pease porridge cold! Pease porridge in the pot nine days old! Some like it hot, some" + + " like it cold, Some like it in the pot nine days old! Pease porridge hot! Pease porridge cold!"; - @Override public void setUp() throws Exception { super.setUp(); - analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); + // Currently only SimpleText and Lucene40 can index offsets into postings: + String codecName = Codec.getDefault().getName(); + assumeTrue("Codec does not support offsets: " + codecName, + codecName.equals("SimpleText") || codecName.equals("Lucene40")); + iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.CREATE); + + if (codecName.equals("Lucene40")) { + // Sep etc are not implemented + switch(random().nextInt(4)) { + case 0: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); break; + case 1: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); break; + case 2: iwc.setCodec(_TestUtil.alwaysPostingsFormat( + new Pulsing40PostingsFormat(_TestUtil.nextInt(random(), 1, 3)))); break; + case 3: iwc.setCodec(_TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat())); break; + } + } + analyzer = iwc.getAnalyzer(); dir = newDirectory(); } - @Override - public void tearDown() throws Exception { - if( searcher != null ){ + public void close() throws IOException { + if (searcher != null) { searcher.getIndexReader().close(); searcher = null; } dir.close(); - super.tearDown(); } // make several docs - protected void insertDocs (Analyzer analyzer, String... values) throws Exception { - IndexWriterConfig config = new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE); - - IndexWriter writer = new IndexWriter(dir, config); - - for( String value: values ) { + protected void insertDocs(Analyzer analyzer, String... values) + throws Exception { + IndexWriter writer = new IndexWriter(dir, iwc); + FieldType type = new FieldType(); + type.setIndexed(true); + type.setTokenized(true); + type.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); + type.setStored(true); + for (String value : values) { Document doc = new Document(); - Field f = new Field (F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS); - doc.add (f); - writer.addDocument( doc ); + Field f = newField(F, value, type); + doc.add(f); + writer.addDocument(doc); } writer.close(); - if( searcher != null ){ + if (searcher != null) { searcher.getIndexReader().close(); } searcher = new IndexSearcher(DirectoryReader.open(dir)); } - private String[] doSearch(Query q) throws IOException, InvalidTokenOffsetsException { + private String[] doSearch(Query q) throws IOException, + InvalidTokenOffsetsException { return doSearch(q, 100); } - private class ConstantScorer implements org.apache.lucene.search.highlight.Scorer { - + private class ConstantScorer implements + org.apache.lucene.search.highlight.Scorer { + @Override public TokenStream init(TokenStream tokenStream) throws IOException { return tokenStream; } - + @Override - public void startFragment(TextFragment newFragment) { - } - + public void startFragment(TextFragment newFragment) {} + @Override public float getTokenScore() { return 1; } - + @Override public float getFragmentScore() { return 1; - } + } } - private String[] doSearch(Query q, int maxFragSize) throws IOException, InvalidTokenOffsetsException { - return doSearch (q, maxFragSize, 0); + private String[] doSearch(Query q, int maxFragSize) throws IOException, + InvalidTokenOffsetsException { + return doSearch(q, maxFragSize, 0); } - private String[] doSearch(Query q, int maxFragSize, int docIndex) throws IOException, InvalidTokenOffsetsException { - // ConstantScorer is a fragment Scorer, not a search result (document) Scorer - Highlighter highlighter = new Highlighter (new ConstantScorer()); + + private String[] doSearch(Query q, int maxFragSize, int docIndex) + throws IOException, InvalidTokenOffsetsException { + // ConstantScorer is a fragment Scorer, not a search result (document) + // Scorer + Highlighter highlighter = new Highlighter(new ConstantScorer()); highlighter.setTextFragmenter(new SimpleFragmenter(maxFragSize)); PosCollector collector = new PosCollector(10); if (q instanceof MultiTermQuery) { - ((MultiTermQuery)q).setRewriteMethod (MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE); + ((MultiTermQuery) q) + .setRewriteMethod(MultiTermQuery.CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE); } searcher.search(q, collector); ScorePosDoc doc = collector.docs[docIndex]; - if (doc == null) - return null; + if (doc == null) return null; String text = searcher.getIndexReader().document(doc.doc).get(F); - PositionOffsetMapper pom = new PositionOffsetMapper (); - // FIXME: test error cases: for non-stored fields, and fields w/no term vectors -// searcher.getIndexReader().getTermFreqVector(doc.doc, F, pom); + // FIXME: test error cases: for non-stored fields, and fields w/no term + // vectors + // searcher.getIndexReader().getTermFreqVector(doc.doc, F, pom); - TextFragment[] fragTexts = highlighter.getBestTextFragments(new PosTokenStream - (text, new PositionIntervalArrayIterator(doc.sortedPositions(), doc.posCount), pom), - text, false, 10); + TextFragment[] fragTexts = highlighter.getBestTextFragments( + new PosTokenStream(text, new PositionIntervalArrayIterator(doc + .sortedPositions(), doc.posCount)), text, false, 10); String[] frags = new String[fragTexts.length]; for (int i = 0; i < frags.length; i++) frags[i] = fragTexts[i].toString(); return frags; } - public void testTerm () throws Exception { + public void testTerm() throws Exception { insertDocs(analyzer, "This is a test test"); - String frags[] = doSearch (new TermQuery(new Term(F, "test"))); - assertEquals ("This is a test test", frags[0]); + String frags[] = doSearch(new TermQuery(new Term(F, "test"))); + assertEquals("This is a test test", frags[0]); + close(); } - public void testSeveralSnippets () throws Exception { - String input = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + - "Let us see what happens to long in this case."; - String gold = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + - "Let us see what happens to long in this case."; + public void testSeveralSnippets() throws Exception { + String input = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + + "Let us see what happens to long in this case."; + String gold = "this is some long text. It has the word long in many places. In fact, it has long on some different fragments. " + + "Let us see what happens to long in this case."; insertDocs(analyzer, input); - String frags[] = doSearch (new TermQuery(new Term(F, "long")), input.length()); - assertEquals (gold, frags[0]); + String frags[] = doSearch(new TermQuery(new Term(F, "long")), + input.length()); + assertEquals(gold, frags[0]); + close(); } - public void testBooleanAnd () throws Exception { + public void testBooleanAnd() throws Exception { insertDocs(analyzer, "This is a test"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.MUST)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.MUST)); - String frags[] = doSearch (bq); - assertEquals ("This is a test", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); } - public void testBooleanAndOtherOrder () throws Exception { + public void testBooleanAndOtherOrder() throws Exception { insertDocs(analyzer, "This is a test"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.MUST)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.MUST)); - String frags[] = doSearch (bq); - assertEquals ("This is a test", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.MUST)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); } - - public void testBooleanOr () throws Exception { + + public void testBooleanOr() throws Exception { insertDocs(analyzer, "This is a test"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.SHOULD)); - String frags[] = doSearch (bq); - assertEquals ("This is a test", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); } - public void testSingleMatchScorer () throws Exception { + public void testSingleMatchScorer() throws Exception { insertDocs(analyzer, "This is a test"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "notoccurringterm")), Occur.SHOULD)); - String frags[] = doSearch (bq); - assertEquals ("This is a test", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "notoccurringterm")), + Occur.SHOULD)); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); } - public void testBooleanNrShouldMatch () throws Exception { + public void testBooleanNrShouldMatch() throws Exception { insertDocs(analyzer, "a b c d e f g h i"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "a")), Occur.SHOULD)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "b")), Occur.SHOULD)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "no")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "a")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "b")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "no")), Occur.SHOULD)); // This generates a ConjunctionSumScorer bq.setMinimumNumberShouldMatch(2); - String frags[] = doSearch (bq); - assertEquals ("a b c d e f g h i", frags[0]); + String frags[] = doSearch(bq); + assertEquals("a b c d e f g h i", frags[0]); // This generates no scorer bq.setMinimumNumberShouldMatch(3); - frags = doSearch (bq); - assertNull (frags); + frags = doSearch(bq); + assertNull(frags); // This generates a DisjunctionSumScorer bq.setMinimumNumberShouldMatch(2); - bq.add(new BooleanClause (new TermQuery(new Term(F, "c")), Occur.SHOULD)); - frags = doSearch (bq); - assertEquals ("a b c d e f g h i", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "c")), Occur.SHOULD)); + frags = doSearch(bq); + assertEquals("a b c d e f g h i", frags[0]); + close(); } public void testPhrase() throws Exception { insertDocs(analyzer, "is it that this is a test, is it"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "is")), Occur.MUST)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "a")), Occur.MUST)); - PositionFilterQuery pfq = new PositionFilterQuery(bq, new BlockPositionIteratorFilter()); - String frags[] = doSearch (pfq); + bq.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "a")), Occur.MUST)); + PositionFilterQuery pfq = new PositionFilterQuery(bq, + new BlockPositionIteratorFilter()); + String frags[] = doSearch(pfq); // make sure we highlight the phrase, and not the terms outside the phrase - assertEquals ("is it that this is a test, is it", frags[0]); + assertEquals("is it that this is a test, is it", frags[0]); + close(); } /* * Failing ... PhraseQuery scorer needs positions()? */ + @Ignore public void testPhraseOriginal() throws Exception { insertDocs(analyzer, "This is a test"); PhraseQuery pq = new PhraseQuery(); pq.add(new Term(F, "a")); pq.add(new Term(F, "test")); - String frags[] = doSearch (pq); - assertEquals ("This is a test", frags[0]); + String frags[] = doSearch(pq); + assertEquals("This is a test", frags[0]); + close(); } - public void testNestedBoolean () throws Exception { + public void testNestedBoolean() throws Exception { insertDocs(analyzer, "This is a test"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "test")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "test")), Occur.SHOULD)); BooleanQuery bq2 = new BooleanQuery(); - bq2.add(new BooleanClause (new TermQuery(new Term(F, "This")), Occur.SHOULD)); - bq2.add(new BooleanClause (new TermQuery(new Term(F, "is")), Occur.SHOULD)); + bq2.add(new BooleanClause(new TermQuery(new Term(F, "This")), Occur.SHOULD)); + bq2.add(new BooleanClause(new TermQuery(new Term(F, "is")), Occur.SHOULD)); bq.add(new BooleanClause(bq2, Occur.SHOULD)); - String frags[] = doSearch (bq); - assertEquals ("This is a test", frags[0]); + String frags[] = doSearch(bq); + assertEquals("This is a test", frags[0]); + close(); } - public void testWildcard () throws Exception { + public void testWildcard() throws Exception { insertDocs(analyzer, "This is a test"); - String frags[] = doSearch (new WildcardQuery(new Term(F, "t*t"))); - assertEquals ("This is a test", frags[0]); + String frags[] = doSearch(new WildcardQuery(new Term(F, "t*t"))); + assertEquals("This is a test", frags[0]); + close(); } public void testMultipleDocumentsAnd() throws Exception { - insertDocs(analyzer, - "This document has no matches", - PORRIDGE_VERSE, + insertDocs(analyzer, "This document has no matches", PORRIDGE_VERSE, "This document has some Pease porridge in it"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "Pease")), Occur.MUST)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "porridge")), Occur.MUST)); - String frags[] = doSearch (bq, 50, 0); - assertEquals ("Pease porridge hot! Pease porridge cold! Pease", frags[0]); - frags = doSearch (bq, 50, 1); - assertEquals ("This document has some Pease porridge in it", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.MUST)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")), Occur.MUST)); + String frags[] = doSearch(bq, 50, 0); + assertEquals( + "Pease porridge hot! Pease porridge cold! Pease", + frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("This document has some Pease porridge in it", + frags[0]); + close(); } - /* - * Failing: need positions callback API since DisjunctionSumScorer consumes all of a doc's - * positions before passing the doc to the collector. - */ + public void testMultipleDocumentsOr() throws Exception { - insertDocs(analyzer, - "This document has no matches", - PORRIDGE_VERSE, + insertDocs(analyzer, "This document has no matches", PORRIDGE_VERSE, "This document has some Pease porridge in it"); BooleanQuery bq = new BooleanQuery(); - bq.add(new BooleanClause (new TermQuery(new Term(F, "Pease")), Occur.SHOULD)); - bq.add(new BooleanClause (new TermQuery(new Term(F, "porridge")), Occur.SHOULD)); - String frags[] = doSearch (bq, 50, 0); - assertEquals ("Pease porridge hot! Pease porridge cold! Pease", frags[0]); - frags = doSearch (bq, 50, 1); - assertEquals ("This document has some Pease porridge in it", frags[0]); + bq.add(new BooleanClause(new TermQuery(new Term(F, "Pease")), Occur.SHOULD)); + bq.add(new BooleanClause(new TermQuery(new Term(F, "porridge")), + Occur.SHOULD)); + String frags[] = doSearch(bq, 50, 0); + assertEquals( + "Pease porridge hot! Pease porridge cold! Pease", + frags[0]); + frags = doSearch(bq, 50, 1); + assertEquals("This document has some Pease porridge in it", + frags[0]); + close(); } + + public static class BlockPositionIteratorFilter implements PositionIntervalFilter { + @Override + public PositionIntervalIterator filter(PositionIntervalIterator iter) { + return new BlockPositionIterator(iter); + } + + } + }