Index: src/test/java/org/apache/jackrabbit/core/query/ExcerptTest.java
===================================================================
--- src/test/java/org/apache/jackrabbit/core/query/ExcerptTest.java (revision 1173196)
+++ src/test/java/org/apache/jackrabbit/core/query/ExcerptTest.java (working copy)
@@ -16,9 +16,9 @@
*/
package org.apache.jackrabbit.core.query;
+import javax.jcr.Node;
import javax.jcr.RepositoryException;
import javax.jcr.Value;
-import javax.jcr.Node;
import javax.jcr.query.QueryResult;
import javax.jcr.query.Row;
import javax.jcr.query.RowIterator;
@@ -116,22 +116,54 @@
QueryResult result = executeQuery(stmt);
RowIterator rows = result.getRows();
assertEquals(1, rows.getSize());
- assertEquals(excerpt, rows.nextRow().getValue("rep:excerpt(text)").getString());
+
+ String ex = getExcerpt(rows.nextRow());
+ assertEquals("Expected " + excerpt + ", but got ", excerpt, ex);
}
public void testEncodeIllegalCharsHighlights() throws RepositoryException {
- String text = "bla bla foo";
- String excerpt = createExcerpt("bla <strong>bla</strong> foo");
- Node n = testRootNode.addNode(nodeName1);
- n.setProperty("text", text);
- superuser.save();
+ checkExcerpt("bla bla foo",
+ "bla <strong>bla</strong> foo",
+ "foo");
+ }
- String stmt = getStatement("foo");
- QueryResult result = executeQuery(stmt);
- RowIterator rows = result.getRows();
- assertEquals(1, rows.getSize());
- assertEquals(excerpt, rows.nextRow().getValue("rep:excerpt(text)").getString());
+ /**
+ * test for https://issues.apache.org/jira/browse/JCR-3077
+ *
+ */
+ public void testQuotedPhraseEn() throws RepositoryException {
+ String content = "one two three four";
+ String terms = "\"two three\"";
+ String expectedExcerpt = "one two three four";
+ checkExcerpt(content, expectedExcerpt, terms);
}
+
+ // TODO test quoted phrase - no match
+ // TODO check quoted phrase - no match if scrambled
+
+ /**
+ * test for https://issues.apache.org/jira/browse/JCR-3077
+ *
+ * JA search acts as a PhraseQuery, thanks to LUCENE-2458. so it should be
+ * covered by the QuotedTest search.
+ *
+ */
+ public void testHighlightJa() throws RepositoryException {
+
+ // http://translate.google.com/#auto|en|%E3%82%B3%E3%83%B3%E3%83%86%E3%83%B3%E3%83%88
+ String jContent = "\u30b3\u30fe\u30c6\u30f3\u30c8";
+ // http://translate.google.com/#auto|en|%E3%83%86%E3%82%B9%E3%83%88
+ String jTest = "\u30c6\u30b9\u30c8";
+
+ String content = "some text with japanese: " + jContent + " (content)"
+ + " and " + jTest + " (test).";
+
+ // expected excerpt; note this may change if excerpt providers change
+ String expectedExcerpt = "some text with japanese: " + jContent
+ + " (content) and " + jTest + " (test).";
+ checkExcerpt(content, expectedExcerpt, jTest);
+ }
+
private void checkExcerpt(String text, String fragmentText, String terms)
throws RepositoryException {
@@ -151,7 +183,7 @@
private void createTestData(String text) throws RepositoryException {
Node n = testRootNode.addNode(nodeName1);
n.setProperty("text", text);
- testRootNode.save();
+ superuser.save();
}
private String getExcerpt(Row row) throws RepositoryException {
Index: src/main/java/org/apache/jackrabbit/core/query/lucene/WeightedHighlighter.java
===================================================================
--- src/main/java/org/apache/jackrabbit/core/query/lucene/WeightedHighlighter.java (revision 1173196)
+++ src/main/java/org/apache/jackrabbit/core/query/lucene/WeightedHighlighter.java (working copy)
@@ -77,7 +77,7 @@
* highlighted
*/
public static String highlight(TermPositionVector tvec,
- Set queryTerms,
+ Set queryTerms,
String text,
String excerptStart,
String excerptEnd,
@@ -103,7 +103,7 @@
* highlighted
*/
public static String highlight(TermPositionVector tvec,
- Set queryTerms,
+ Set queryTerms,
String text,
int maxFragments,
int surround) throws IOException {
@@ -129,7 +129,7 @@
fragmentStart, fragmentEnd, surround * 2);
}
- PriorityQueue bestFragments = new FragmentInfoPriorityQueue(maxFragments);
+ PriorityQueue bestFragments = new FragmentInfoPriorityQueue(maxFragments);
for (int i = 0; i < offsets.length; i++) {
if (offsets[i].getEndOffset() <= text.length()) {
FragmentInfo fi = new FragmentInfo(offsets[i], surround * 2);
@@ -336,7 +336,7 @@
}
- private static class FragmentInfoPriorityQueue extends PriorityQueue {
+ private static class FragmentInfoPriorityQueue extends PriorityQueue {
public FragmentInfoPriorityQueue(int size) {
initialize(size);
@@ -350,9 +350,7 @@
* {@link FragmentInfo} with the best quality.
*/
@Override
- protected boolean lessThan(Object a, Object b) {
- FragmentInfo infoA = (FragmentInfo) a;
- FragmentInfo infoB = (FragmentInfo) b;
+ protected boolean lessThan(FragmentInfo infoA, FragmentInfo infoB) {
if (infoA.getQuality() == infoB.getQuality()) {
return infoA.getStartOffset() > infoB.getStartOffset();
}
Index: src/main/java/org/apache/jackrabbit/core/query/lucene/DefaultHighlighter.java
===================================================================
--- src/main/java/org/apache/jackrabbit/core/query/lucene/DefaultHighlighter.java (revision 1173196)
+++ src/main/java/org/apache/jackrabbit/core/query/lucene/DefaultHighlighter.java (working copy)
@@ -19,17 +19,19 @@
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.Set;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.jackrabbit.util.Text;
import org.apache.lucene.document.Field;
+import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.index.TermVectorOffsetInfo;
-import org.apache.lucene.index.Term;
-import org.apache.jackrabbit.util.Text;
/**
* This is an adapted version of the FulltextHighlighter posted in
@@ -93,7 +95,7 @@
* highlighted
*/
public static String highlight(TermPositionVector tvec,
- Set queryTerms,
+ Set queryTerms,
String text,
String excerptStart,
String excerptEnd,
@@ -120,7 +122,7 @@
* highlighted
*/
public static String highlight(TermPositionVector tvec,
- Set queryTerms,
+ Set queryTerms,
String text,
int maxFragments,
int surround)
@@ -134,7 +136,7 @@
* @see #highlight(TermPositionVector, Set, String, String, String, String, String, String, String, int, int)
*/
protected String doHighlight(TermPositionVector tvec,
- Set queryTerms,
+ Set queryTerms,
String text,
String excerptStart,
String excerptEnd,
@@ -144,21 +146,103 @@
String hlEnd,
int maxFragments,
int surround) throws IOException {
- String[] terms = new String[queryTerms.size()];
- Iterator it = queryTerms.iterator();
- for (int i = 0; it.hasNext(); i++) {
- terms[i] = it.next().text();
- }
- List list = new ArrayList();
- int[] tvecindexes = tvec.indexesOf(terms, 0, terms.length);
- for (int tvecindex : tvecindexes) {
- TermVectorOffsetInfo[] termoffsets = tvec.getOffsets(tvecindex);
- list.addAll(Arrays.asList(termoffsets));
+
+ List termOffsetInfo = new ArrayList();
+
+ Iterator it = queryTerms.iterator();
+ while (it.hasNext()) {
+ Term[] qt = it.next();
+ final int qtLen = qt.length;
+ if (qt == null || qtLen == 0) {
+ continue;
+ }
+ String[] qtText = new String[qtLen];
+ for (int i = 0; i < qtLen; i++) {
+ qtText[i] = qt[i].text();
+ }
+ int[] tvecindexes = tvec.indexesOf(qtText, 0, qtText.length);
+ Map localTermOffsetInfo = new HashMap();
+ for (int tvecindex : tvecindexes) {
+ TermVectorOffsetInfo[] termoffsets = tvec.getOffsets(tvecindex);
+ if (termoffsets == null || termoffsets.length == 0) {
+ continue;
+ }
+ localTermOffsetInfo.put(tvecindex, termoffsets);
+ }
+
+ //to keep the order of the keys, use tvecindexes
+ if (tvecindexes.length > 1) {
+ // we have to build one interval TermVectorOffsetInfo for each
+ // hit;
+ List intervalTermOffsetInfo = new ArrayList();
+
+ // pick all the first key's hist as interval start
+ TermVectorOffsetInfo[] firstKeyTermOffsets = localTermOffsetInfo
+ .get(tvecindexes[0]);
+ Arrays.sort(firstKeyTermOffsets,
+ new TermVectorOffsetInfoSorter());
+ intervalTermOffsetInfo.addAll(Arrays
+ .asList(firstKeyTermOffsets));
+
+ // check if each key is part of an interval, if not, it is
+ // dropped from the list
+ for (int i = 1; i < tvecindexes.length; i++) {
+ final Integer key = tvecindexes[i];
+ TermVectorOffsetInfo[] termoffsets = localTermOffsetInfo
+ .get(key);
+ if (termoffsets == null) {
+ continue;
+ }
+ Arrays.sort(termoffsets, new TermVectorOffsetInfoSorter());
+
+ Iterator intervalIterator = intervalTermOffsetInfo
+ .iterator();
+
+ int index = 0;
+ while (intervalIterator.hasNext()) {
+ TermVectorOffsetInfo intervalOI = intervalIterator
+ .next();
+ if (index >= termoffsets.length) {
+ intervalIterator.remove();
+ continue;
+ }
+ boolean matchSearch = true;
+ boolean matchFound = false;
+ while (matchSearch) {
+ TermVectorOffsetInfo localOI = termoffsets[index];
+ // check interval match
+ // CJK languages will have the tokens from the PhraseQuery glued together (see LUCENE-2458)
+ int diff = localOI.getStartOffset()
+ - intervalOI.getEndOffset();
+ // TODO we'll probably have to remove 'diff == 0'
+ // after upgrading to lucene 3.1
+ if (diff == 1 || diff == 0) {
+ intervalOI.setEndOffset(localOI.getEndOffset());
+ matchSearch = false;
+ matchFound = true;
+ }
+ index++;
+ if (index >= termoffsets.length) {
+ matchSearch = false;
+ }
+ }
+ if (!matchFound) {
+ index--;
+ intervalIterator.remove();
+ }
+ }
+ }
+ termOffsetInfo.addAll(intervalTermOffsetInfo);
+
+ } else if (tvecindexes.length == 1) {
+ termOffsetInfo.addAll(Arrays.asList(localTermOffsetInfo
+ .values().iterator().next()));
+ }
}
- TermVectorOffsetInfo[] offsets = list.toArray(new TermVectorOffsetInfo[list.size()]);
+ TermVectorOffsetInfo[] offsets = termOffsetInfo.toArray(new TermVectorOffsetInfo[termOffsetInfo.size()]);
// sort offsets
- if (terms.length > 1) {
+ if (offsets != null && offsets.length > 1) {
Arrays.sort(offsets, new TermVectorOffsetInfoSorter());
}
Index: src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractExcerpt.java
===================================================================
--- src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractExcerpt.java (revision 1154949)
+++ src/main/java/org/apache/jackrabbit/core/query/lucene/AbstractExcerpt.java (working copy)
@@ -18,6 +18,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
@@ -35,6 +37,7 @@
import java.io.IOException;
import java.io.StringReader;
import java.io.Reader;
+import java.util.Iterator;
import java.util.Set;
import java.util.HashSet;
import java.util.TreeMap;
@@ -177,10 +180,32 @@
/**
* @return the extracted terms from the query.
*/
- protected final Set getQueryTerms() {
+ protected final Set getQueryTerms() {
+ Set relevantTerms = new HashSet();
+ getQueryTerms(query, relevantTerms);
+ return relevantTerms;
+ }
+
+ private static void getQueryTerms(Query q, Set relevantTerms) {
+ if (q instanceof BooleanQuery) {
+ final BooleanQuery bq = (BooleanQuery) q;
+ Iterator clIterator = bq.iterator();
+ while (clIterator.hasNext()) {
+ BooleanClause clause = clIterator.next();
+ getQueryTerms(clause.getQuery(), relevantTerms);
+ }
+ return;
+ }
Set extractedTerms = new HashSet();
+ q.extractTerms(extractedTerms);
+ Set filteredTerms = filterRelevantTerms(extractedTerms);
+ if (!filteredTerms.isEmpty()) {
+ relevantTerms.add(filteredTerms.toArray(new Term[] {}));
+ }
+ }
+
+ private static Set filterRelevantTerms(Set extractedTerms) {
Set relevantTerms = new HashSet();
- query.extractTerms(extractedTerms);
// only keep terms for fulltext fields
for (Term t : extractedTerms) {
if (t.field().equals(FieldNames.FULLTEXT)) {