Index: lucene/contrib/grouping/README.txt =================================================================== --- lucene/contrib/grouping/README.txt (revision ) +++ lucene/contrib/grouping/README.txt (revision ) @@ -0,0 +1,1 @@ +contrib/grouping is a home of grouping functionality \ No newline at end of file Index: lucene/contrib/grouping/pom.xml.template =================================================================== --- lucene/contrib/grouping/pom.xml.template (revision ) +++ lucene/contrib/grouping/pom.xml.template (revision ) @@ -0,0 +1,36 @@ + + + + + 4.0.0 + + org.apache.lucene + lucene-contrib + @version@ + + org.apache.lucene + lucene-grouping + Lucene Grouping + @version@ + Lucene Grouping extensions + jar + Index: lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupDoc.java =================================================================== --- lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupDoc.java (revision ) +++ lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupDoc.java (revision ) @@ -0,0 +1,39 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +/** + * Represents a collapse group. Contains the most relevant document (aka dochead) with its score and the number + * of documents that have the same field value on a predefined field but are inferior. + */ +public class GroupDoc extends ScoreDoc { + + /** + * The number of inferior documents . + * In general this total number of documents - collapse threshold. + */ + public int groupCount; + + /** + * @param doc Most relevant document of this group + * @param score The score of the most relevant document of this group + */ + protected GroupDoc(int doc, float score) { + super(doc, score); + } + +} Index: lucene/contrib/grouping/src/test/org/apache/lucene/search/CollapseCollectorIntegrationTest.java =================================================================== --- lucene/contrib/grouping/src/test/org/apache/lucene/search/CollapseCollectorIntegrationTest.java (revision ) +++ lucene/contrib/grouping/src/test/org/apache/lucene/search/CollapseCollectorIntegrationTest.java (revision ) @@ -0,0 +1,113 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import org.apache.lucene.analysis.core.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; + +public class CollapseCollectorIntegrationTest extends LuceneTestCase { + + private static final Version LUCENE_VERSION = Version.LUCENE_40; + + private IndexSearcher indexSearcher; + + @Override + protected void setUp() throws Exception { + super.setUp(); + RAMDirectory directory = new RAMDirectory(); + IndexWriter writer = new IndexWriter( + directory, + new IndexWriterConfig(LUCENE_VERSION, new WhitespaceAnalyzer(LUCENE_VERSION)) + ); + + Document document1 = new Document(); + document1.add(new Field("author", "author1", Field.Store.YES, Field.Index.ANALYZED)); + document1.add(new Field("content", "random text", Field.Store.YES, Field.Index.ANALYZED)); + document1.add(new Field("id", "1", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document1); + writer.commit(); + + Document document2 = new Document(); + document2.add(new Field("author", "author1", Field.Store.YES, Field.Index.ANALYZED)); + document2.add(new Field("content", "some more random text", Field.Store.YES, Field.Index.ANALYZED)); + document2.add(new Field("id", "2", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document2); + + Document document3 = new Document(); + document3.add(new Field("author", "author1", Field.Store.YES, Field.Index.ANALYZED)); + document3.add(new Field("content", "some more random textual data", Field.Store.YES, Field.Index.ANALYZED)); + document3.add(new Field("id", "3", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document3); + + Document document4 = new Document(); + document4.add(new Field("author", "author2", Field.Store.YES, Field.Index.ANALYZED)); + document4.add(new Field("content", "some random text", Field.Store.YES, Field.Index.ANALYZED)); + document4.add(new Field("id", "4", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document4); + + Document document5 = new Document(); + document5.add(new Field("author", "author3", Field.Store.YES, Field.Index.ANALYZED)); + document5.add(new Field("content", "some more random text", Field.Store.YES, Field.Index.ANALYZED)); + document5.add(new Field("id", "5", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document5); + + Document document6 = new Document(); + document6.add(new Field("author", "author3", Field.Store.YES, Field.Index.ANALYZED)); + document6.add(new Field("content", "random", Field.Store.YES, Field.Index.ANALYZED)); + document6.add(new Field("id", "6", Field.Store.YES, Field.Index.NO)); + writer.addDocument(document6); + writer.commit(); + writer.optimize(); + + indexSearcher = new IndexSearcher(directory); + } + + public void testCollapse() throws Exception { + GroupCollector groupCollector = new GroupCollector("author", 1, 10, indexSearcher, Sort.RELEVANCE); // sorting is score + indexSearcher.search(new TermQuery(new Term("content", "random")), groupCollector); + TopDocs groups = groupCollector.getTopGroups(); + + assertEquals(3, groups.totalHits); + assertEquals(3, groups.scoreDocs.length); + assertEquals(groups.getMaxScore(), groups.scoreDocs[0].score); + + // the later a document is added the higher this docId value + assertEquals(5, groups.scoreDocs[0].doc); + assertEquals(1, ((GroupDoc)groups.scoreDocs[0]).groupCount); + assertEquals("author3", indexSearcher.doc(groups.scoreDocs[0].doc).get("author")); + assertEquals("6", indexSearcher.doc(groups.scoreDocs[0].doc).get("id")); + assertEquals(0, groups.scoreDocs[1].doc); + assertEquals(2, ((GroupDoc)groups.scoreDocs[1]).groupCount); + assertEquals("author1", indexSearcher.doc(groups.scoreDocs[1].doc).get("author")); + assertEquals("1", indexSearcher.doc(groups.scoreDocs[1].doc).get("id")); + assertEquals(3, groups.scoreDocs[2].doc); + assertEquals(0, ((GroupDoc)groups.scoreDocs[2]).groupCount); + assertEquals("author2", indexSearcher.doc(groups.scoreDocs[2].doc).get("author")); + assertEquals("4", indexSearcher.doc(groups.scoreDocs[2].doc).get("id")); + + assertTrue(groups.scoreDocs[0].score > groups.scoreDocs[1].score); + assertTrue(groups.scoreDocs[1].score > groups.scoreDocs[2].score); + } + +} \ No newline at end of file Index: lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupCollector.java =================================================================== --- lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupCollector.java (revision ) +++ lucene/contrib/grouping/src/java/org/apache/lucene/search/GroupCollector.java (revision ) @@ -0,0 +1,374 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.PriorityQueue; + +import java.io.IOException; +import java.util.*; + +/** + * The group collector groups collected documents by field value and counts . + */ +public class GroupCollector extends Collector { + + private final FieldCache.DocTermsIndex groupFieldValues; + private final int maxNumberOfGroups; + private final int groupThreshold; + private final Map groupedDocs; + + private int docBase; + private int unGroupedDocCount; + + private boolean groupQueueFull = false; + private NonAdjacentGroup bottomGroup; + + private final FieldComparator[] fieldComparators; + private final int[] descending; + private Scorer scorer; + private final PriorityQueue groupResultPriorityQueue; + + /** + * @param groupField The name of the field to group to collected document + * @param groupThreshold The threshold per unique group to start grouping. If a group has less similar documents + * than this threshold no grouping will occur + * @param maxNumberOfGroups The maximum number of groups to return as result + * @param searcher The index searcher + * @param sort The sort + * @throws IOException + */ + public GroupCollector(String groupField, + int groupThreshold, + int maxNumberOfGroups, + IndexSearcher searcher, + Sort sort) throws IOException { + + int maxDoc = searcher.maxDoc(); + this.groupThreshold = groupThreshold; + this.maxNumberOfGroups = maxNumberOfGroups; + // When collapsing a large result set, a lot of time is wasted resizing this hash map. Therefore I initialize + // it with size equals to maxDoc. Some more memory may be taken than necessary... + this.groupedDocs = new HashMap(maxDoc); + groupFieldValues = FieldCache.DEFAULT.getTermsIndex(searcher.getIndexReader(), groupField); + + fieldComparators = new FieldComparator[sort.fields.length]; + descending = new int[sort.fields.length]; + for (int i = 0; i < sort.getSort().length; i++) { + SortField sortField = sort.getSort()[i]; + fieldComparators[i] = sortField.getComparator(maxDoc, i); + descending[i] = sortField.getReverse() ? -1 : 1; + } + + groupResultPriorityQueue = new GroupPriorityQueue(maxNumberOfGroups, fieldComparators, descending); + } + + /** + * {@inheritDoc} + */ + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + for (FieldComparator fieldComparator : fieldComparators) { + fieldComparator.setScorer(scorer); + } + } + + /** + * {@inheritDoc} + */ + public void collect(int doc) throws IOException { + ++unGroupedDocCount; + doc += docBase; + doGrouping(doc); + } + + /** + * {@inheritDoc} + */ + public void setNextReader(IndexReader reader, int docBase) throws IOException { + this.docBase = docBase; + for (FieldComparator fieldComparator : fieldComparators) { + fieldComparator.setNextReader(reader, docBase); + } + } + + /** + * {@inheritDoc} + */ + public boolean acceptsDocsOutOfOrder() { + return true; + } + + /** + * Returns the top groups collected by this collector. + * + * @return the top groups collected by this collector + * @throws IOException If io related exception occurs during creating the top groups + */ + public TopDocs getTopGroups() throws IOException { + for (NonAdjacentGroup group : groupedDocs.values()) { + sortGroup(group); + } + + int length = + groupResultPriorityQueue.size() < maxNumberOfGroups ? groupResultPriorityQueue.size() : maxNumberOfGroups; + GroupDoc[] topCollapseGroups = new GroupDoc[length]; + + for (int i = 0; i < length; i++) { + NonAdjacentGroup group = groupResultPriorityQueue.pop(); + topCollapseGroups[i] = group; + } + + float maxScore = length > 0 ? topCollapseGroups[0].score : 0; + return new TopDocs(groupedDocs.size(), topCollapseGroups, maxScore); + } + + + // ================================================= Helpers ========================================================= + + protected void doGrouping(int currentId) throws IOException { + int fieldValueOrd = groupFieldValues.getOrd(currentId); + if (fieldValueOrd == 0) { + return; + } + + // Get the last doc. and the total amount of docs. we have seen so + // far for this group value + NonAdjacentGroup doc = groupedDocs.get(fieldValueOrd); + int slot = unGroupedDocCount - 1; + if (doc == null) { + // new group value => create a new record for it + doc = new NonAdjacentGroup(fieldValueOrd, fieldComparators, descending, scorer, currentId, slot, groupThreshold); + groupedDocs.put(fieldValueOrd, doc); + } else { + doc.setScorer(scorer); + doc.compare(currentId, slot); + } + + // check if we have reached the group threshold, if so start counting inferior grouped documents + if (++doc.totalCount > groupThreshold) { + doc.groupCount++; + } + } + + /** + * Potentially adds a group to the result. The specified group is sorted based on the relevancy of the most relevant + * document of the specified group. The specified group can be irrelevant and thus be neglected. + * + * @param group The specified group + * @throws IOException + */ + void sortGroup(NonAdjacentGroup group) throws IOException { + if (groupQueueFull) { + for (int i = 0; i < fieldComparators.length; i++) { + int result = descending[i] * fieldComparators[i].compare(bottomGroup.bottomDoc.slot, group.bottomDoc.slot); + if (result < 0) { + return; + } else if (result > 0) { + break; + } else if (i == (fieldComparators.length - 1) && group.bottomDoc.docId <= bottomGroup.bottomDoc.docId) { + return; + } + } + + bottomGroup = groupResultPriorityQueue.insertWithOverflow(group); + } else { + bottomGroup = groupResultPriorityQueue.add(group); + groupQueueFull = groupResultPriorityQueue.size() == maxNumberOfGroups; + } + } + + + // ============================================ Inner Classes ======================================================== + + static class RelevantDocumentPriorityQueue extends PriorityQueue { + + private final FieldComparator[] fieldComparators; + private final int[] descending; + + RelevantDocumentPriorityQueue(int collapseThreshold, FieldComparator[] fieldComparators, int[] descending) { + initialize(collapseThreshold); + this.fieldComparators = fieldComparators; + this.descending = descending; + } + + + /** + * {@inheritDoc} + */ + protected boolean lessThan(RelevantDocument a, RelevantDocument b) { + for (int i = 0; i < fieldComparators.length; i++) { + final int result = descending[i] * fieldComparators[i].compare(a.slot, b.slot); + + if (result != 0) { + return result > 0; + } + } + + return a.docId > b.docId; + } + + } + + static class GroupPriorityQueue extends PriorityQueue { + + private final FieldComparator[] fieldComparators; + private final int[] descending; + + GroupPriorityQueue(int maxGroups, FieldComparator[] fieldComparators, int[] descending) { + initialize(maxGroups); + this.fieldComparators = fieldComparators; + this.descending = descending; + } + + /** + * {@inheritDoc} + */ + protected boolean lessThan(NonAdjacentGroup groupA, NonAdjacentGroup groupB) { + RelevantDocument groupAHeadSlot = groupA.priorityQueue.top(); + RelevantDocument groupBHeadSlot = groupB.priorityQueue.top(); + + for (int i = 0; i < fieldComparators.length; i++) { + int result = descending[i] * fieldComparators[i].compare(groupAHeadSlot.slot, groupBHeadSlot.slot); + + if (result != 0) { + return result < 0; + } + } + + return groupAHeadSlot.docId > groupBHeadSlot.docId; + } + + } + + /** + * Represents a group, that groups documents based on field value irrelevant of the document's order in the result set. + *

+ * Keeps track of the following statistics during the collapsing on fieldvalue: + *

    + *
  • how many documents were collapsed + *
  • what the most relevant head documents are in this group (these will not get collapsed) + *
  • how many documents have been processed + *
+ */ + static class NonAdjacentGroup extends GroupDoc { + + int totalCount = 0; + + boolean queueFull; + RelevantDocument bottomDoc; + + PriorityQueue priorityQueue; + int fieldValueOrd; + + private FieldComparator[] fieldComparators; + private int[] descending; + private int collapseThreshold; + + private Scorer scorer; + + NonAdjacentGroup(int fieldValueOrd, FieldComparator[] fieldComparators, int[] descending, Scorer scorer, int docId, int slot, int collapseThreshold) throws IOException { + super(docId, scorer.score()); + + this.fieldValueOrd = fieldValueOrd; + this.fieldComparators = fieldComparators; + this.descending = descending; + this.scorer = scorer; + this.priorityQueue = new RelevantDocumentPriorityQueue(collapseThreshold, fieldComparators, descending); + this.collapseThreshold = collapseThreshold; + for (FieldComparator fieldComparator : fieldComparators) { + fieldComparator.copy(slot, docId); + } + bottomDoc = priorityQueue.add(new RelevantDocument(docId, slot)); + queueFull = priorityQueue.size() == collapseThreshold; + } + + /** + * Compares if the specified document is more relevant than the document head of this group. + * + * @param docId The id of the specified document + * @param slot The slot dedicated to the specified document (used by field comparators to store values into) + * @throws IOException If I/O related error occur + */ + void compare(int docId, int slot) throws IOException { + if (queueFull) { + for (int i = 0; i < fieldComparators.length; i++) { + fieldComparators[i].copy(slot, docId); + int result = descending[i] * fieldComparators[i].compare(this.bottomDoc.slot, slot); + + if (result < 0) { + return; + } else if (result > 0) { + break; + } else if (i == (fieldComparators.length - 1) && docId <= bottomDoc.docId) { + return; + } + } + + bottomDoc = priorityQueue.insertWithOverflow(new RelevantDocument(docId, slot)); + this.doc = priorityQueue.top().docId; + this.score = scorer.score(); + } else { + priorityQueue.add(new RelevantDocument(docId, slot)); + queueFull = priorityQueue.size() == collapseThreshold; + + this.doc = priorityQueue.top().docId; + this.score = scorer.score(); + } + } + + void setScorer(Scorer scorer) { + this.scorer = scorer; + } + + @Override + public int hashCode() { + return fieldValueOrd; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NonAdjacentGroup that = (NonAdjacentGroup) o; + return fieldValueOrd == that.fieldValueOrd; + } + + @Override + public String toString() { + return "NonAdjacentGroup{" + + "collapseCount=" + groupCount + + ", docHead=" + doc + + '}'; + } + } + + + static class RelevantDocument { + + final int docId; + final int slot; + + RelevantDocument(int docId, int slot) { + this.docId = docId; + this.slot = slot; + } + + } + +} Index: lucene/contrib/grouping/build.xml =================================================================== --- lucene/contrib/grouping/build.xml (revision ) +++ lucene/contrib/grouping/build.xml (revision ) @@ -0,0 +1,27 @@ + + + + + + + + Grouping Lucene extensions + + + + \ No newline at end of file