This results in faster performance (~25% faster QPS)
+ * than the two-pass grouping collectors, with the tradeoff
+ * being that the documents in each group must always be
+ * indexed as a block.
+ *
+ *
See {@link org.apache.lucene.search.grouping} for more
+ * details including a full code example.
+ *
+ * @lucene.experimental
+ */
+
+public class DocBlockGroupingCollector extends Collector {
+
+ private int[] pendingSubDocs;
+ private float[] pendingSubScores;
+ private int subDocUpto;
+
+ private final Sort groupSort;
+ private final Sort withinGroupSort;
+ private final int topNGroups;
+ private final int maxDocsPerGroup;
+ private final Filter lastDocPerGroup;
+
+ // TODO: specialize into 2 classes, static "create" method:
+ private final boolean needsScores;
+
+ private final FieldComparator[] comparators;
+ private final int[] reversed;
+ private final int compIDXEnd;
+ private int bottomSlot;
+ private boolean queueFull;
+ private AtomicReaderContext currentReaderContext;
+
+ private int topGroupDoc;
+ private int totalHitCount;
+ private int docBase;
+ private int groupEndDocID;
+ //private OpenBitSet lastDocPerGroupBits;
+ private DocIdSetIterator lastDocPerGroupBits;
+ private Scorer scorer;
+ private final GroupQueue groupQueue;
+ private boolean groupCompetes;
+
+ private final static class FakeScorer extends Scorer {
+
+ float score;
+ int doc;
+
+ public FakeScorer() {
+ super((Weight) null);
+ }
+
+ @Override
+ public float score() {
+ return score;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int nextDoc() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static final class OneGroup {
+ AtomicReaderContext readerContext;
+ //int groupOrd;
+ int topGroupDoc;
+ int[] docs;
+ float[] scores;
+ int count;
+ int comparatorSlot;
+ }
+
+ // Sorts by groupSort. Not static -- uses comparators, reversed
+ private final class GroupQueue extends PriorityQueue {
+
+ public GroupQueue(int size) {
+ super(size);
+ }
+
+ @Override
+ protected boolean lessThan(final OneGroup group1, final OneGroup group2) {
+
+ //System.out.println(" ltcheck");
+ assert group1 != group2;
+ assert group1.comparatorSlot != group2.comparatorSlot;
+
+ final int numComparators = comparators.length;
+ for (int compIDX=0;compIDX < numComparators; compIDX++) {
+ final int c = reversed[compIDX] * comparators[compIDX].compare(group1.comparatorSlot, group2.comparatorSlot);
+ if (c != 0) {
+ // Short circuit
+ return c > 0;
+ }
+ }
+
+ // Break ties by docID; lower docID is always sorted first
+ return group1.topGroupDoc > group2.topGroupDoc;
+ }
+ }
+
+ /*
+ private String getGroupString(int groupOrd) {
+ if (groupOrd == 0) {
+ return null;
+ } else {
+ return index.lookup(groupOrd, new BytesRef()).utf8ToString();
+ }
+ }
+ */
+
+ // Called when we transition to another group; if the
+ // group is competitive we insert into the group queue
+ private void processGroup() {
+ //System.out.println(" processGroup ord=" + lastGroupOrd + " competes=" + groupCompetes + " count=" + subDocUpto + " groupDoc=" + topGroupDoc);
+ if (groupCompetes) {
+ if (!queueFull) {
+ // Startup transient: always add a new OneGroup
+ final OneGroup og = new OneGroup();
+ og.count = subDocUpto;
+ og.topGroupDoc = docBase + topGroupDoc;
+ og.docs = new int[subDocUpto];
+ System.arraycopy(pendingSubDocs, 0, og.docs, 0, subDocUpto);
+ if (needsScores) {
+ og.scores = new float[subDocUpto];
+ System.arraycopy(pendingSubScores, 0, og.scores, 0, subDocUpto);
+ }
+ og.readerContext = currentReaderContext;
+ //og.groupOrd = lastGroupOrd;
+ og.comparatorSlot = bottomSlot;
+ final OneGroup bottomGroup = groupQueue.add(og);
+ //System.out.println(" ADD group=" + getGroupString(lastGroupOrd) + " newBottom=" + getGroupString(bottomGroup.groupOrd));
+ queueFull = groupQueue.size() == topNGroups;
+ if (queueFull) {
+ // Queue just became full; now set the real bottom
+ // in the comparators:
+ bottomSlot = bottomGroup.comparatorSlot;
+ //System.out.println(" set bottom=" + bottomSlot);
+ for (int i = 0; i < comparators.length; i++) {
+ comparators[i].setBottom(bottomSlot);
+ }
+ //System.out.println(" QUEUE FULL");
+ } else {
+ // Queue not full yet -- just advance bottomSlot:
+ bottomSlot = groupQueue.size();
+ }
+ } else {
+ // Replace bottom element in PQ and then updateTop
+ final OneGroup og = groupQueue.top();
+ assert og != null;
+ og.count = subDocUpto;
+ og.topGroupDoc = docBase + topGroupDoc;
+ if (og.docs.length < subDocUpto) {
+ og.docs = ArrayUtil.grow(og.docs, subDocUpto);
+ }
+ System.arraycopy(pendingSubDocs, 0, og.docs, 0, subDocUpto);
+ if (needsScores) {
+ if (og.scores.length < subDocUpto) {
+ og.scores = ArrayUtil.grow(og.scores, subDocUpto);
+ }
+ System.arraycopy(pendingSubScores, 0, og.scores, 0, subDocUpto);
+ }
+ og.readerContext = currentReaderContext;
+ //og.groupOrd = lastGroupOrd;
+ bottomSlot = groupQueue.updateTop().comparatorSlot;
+
+ //System.out.println(" set bottom=" + bottomSlot);
+ for (int i = 0; i < comparators.length; i++) {
+ comparators[i].setBottom(bottomSlot);
+ }
+ }
+ }
+ subDocUpto = 0;
+ }
+
+ /**
+ * Create the single pass collector.
+ *
+ * @param groupSort The {@link Sort} used to sort the
+ * groups. The top sorted document within each group
+ * according to groupSort, determines how that group
+ * sorts against other groups. This must be non-null,
+ * ie, if you want to groupSort by relevance use
+ * Sort.RELEVANCE.
+ * @param topNGroups How many top groups to keep.
+ * @param withinGroupSort The {@link Sort} used to sort
+ * documents within each group. Passing null is
+ * allowed, to sort by relevance.
+ * @param maxDocsPerGroup How many top documents to keep
+ * within each group.
+ * @param needsScores true if the collected documents
+ * require scores, either because relevance is included
+ * in the withinGroupSort or because you plan to pass true
+ * for either getSscores or getMaxScores to {@link
+ * #getTopGroups}
+ * @param lastDocPerGroup a {@link Filter} that marks the
+ * last document in each group.
+ */
+ public DocBlockGroupingCollector(Sort groupSort, int topNGroups, Sort withinGroupSort, int maxDocsPerGroup, boolean needsScores, Filter lastDocPerGroup) throws IOException {
+
+ if (topNGroups < 1) {
+ throw new IllegalArgumentException("topNGroups must be >= 1 (got " + topNGroups + ")");
+ }
+
+ groupQueue = new GroupQueue(topNGroups);
+ pendingSubDocs = new int[10];
+ if (needsScores) {
+ pendingSubScores = new float[10];
+ }
+
+ this.maxDocsPerGroup = maxDocsPerGroup;
+ this.withinGroupSort = withinGroupSort;
+ this.needsScores = needsScores;
+ this.lastDocPerGroup = lastDocPerGroup;
+ // TODO: allow null groupSort to mean "by relevance",
+ // and specialize it?
+ this.groupSort = groupSort;
+
+ this.topNGroups = topNGroups;
+
+ final SortField[] sortFields = groupSort.getSort();
+ comparators = new FieldComparator[sortFields.length];
+ compIDXEnd = comparators.length - 1;
+ reversed = new int[sortFields.length];
+ for (int i = 0; i < sortFields.length; i++) {
+ final SortField sortField = sortFields[i];
+ comparators[i] = sortField.getComparator(topNGroups, i);
+ reversed[i] = sortField.getReverse() ? -1 : 1;
+ }
+ }
+
+ // TODO: maybe allow no sort on retrieving groups? app
+ // may want to simply process docs in the group itself?
+ // typically they will be presented as a "single" result
+ // in the UI?
+
+ /** Returns the grouped results. Returns null if the
+ * number of groups collected is <= groupOffset.
+ *
+ * NOTE: This collector is unable to compute
+ * the groupValue per group so it will always be null.
+ * This is normally not a problem, as you can obtain the
+ * value just like you obtain other values for each
+ * matching document (eg, via stored fields, via
+ * FieldCache, etc.)
+ */
+ public TopGroups getTopGroups(int groupOffset, int withinGroupOffset, boolean fillGroupValues, boolean fillSortFields, boolean getScores, boolean getMaxScores) throws IOException {
+ //if (queueFull) {
+ //System.out.println("getTopGroups groupOffset=" + groupOffset + " topNGroups=" + topNGroups);
+ //}
+ if (subDocUpto != 0) {
+ processGroup();
+ }
+ if (groupOffset >= groupQueue.size()) {
+ return null;
+ }
+ int totalGroupedHitCount = 0;
+
+ final FakeScorer fakeScorer = new FakeScorer();
+
+ final GroupDocs[] groups = new GroupDocs[groupQueue.size() - groupOffset];
+ for(int downTo=groupQueue.size()-groupOffset-1;downTo>=0;downTo--) {
+ final OneGroup og = groupQueue.pop();
+
+ // At this point we hold all docs w/ in each group,
+ // unsorted; we now sort them:
+ final TopDocsCollector collector;
+ if (withinGroupSort == null) {
+ // Sort by score
+ collector = TopScoreDocCollector.create(maxDocsPerGroup, true);
+ } else {
+ // Sort by fields
+ collector = TopFieldCollector.create(withinGroupSort, maxDocsPerGroup, fillSortFields, getScores, getMaxScores, true);
+ }
+
+ collector.setScorer(fakeScorer);
+ collector.setNextReader(og.readerContext);
+ for(int docIDX=0;docIDX groupEndDocID) {
+ // Group changed
+ if (subDocUpto != 0) {
+ processGroup();
+ }
+ groupEndDocID = lastDocPerGroupBits.advance(doc);
+ //System.out.println(" adv " + groupEndDocID + " " + lastDocPerGroupBits);
+ subDocUpto = 0;
+ groupCompetes = !queueFull;
+ }
+
+ totalHitCount++;
+
+ // Always cache doc/score within this group:
+ if (subDocUpto == pendingSubDocs.length) {
+ pendingSubDocs = ArrayUtil.grow(pendingSubDocs);
+ if (needsScores) {
+ pendingSubScores = ArrayUtil.grow(pendingSubScores);
+ }
+ }
+ pendingSubDocs[subDocUpto] = doc;
+ if (needsScores) {
+ pendingSubScores[subDocUpto] = scorer.score();
+ }
+ subDocUpto++;
+
+ if (groupCompetes) {
+ if (subDocUpto == 1) {
+ assert !queueFull;
+
+ //System.out.println(" init copy to bottomSlot=" + bottomSlot);
+ for (FieldComparator fc : comparators) {
+ fc.copy(bottomSlot, doc);
+ fc.setBottom(bottomSlot);
+ }
+ topGroupDoc = doc;
+ } else {
+ // Compare to bottomSlot
+ for (int compIDX = 0;; compIDX++) {
+ final int c = reversed[compIDX] * comparators[compIDX].compareBottom(doc);
+ if (c < 0) {
+ // Definitely not competitive -- done
+ return;
+ } else if (c > 0) {
+ // Definitely competitive.
+ break;
+ } else if (compIDX == compIDXEnd) {
+ // Ties with bottom, except we know this docID is
+ // > docID in the queue (docs are visited in
+ // order), so not competitive:
+ return;
+ }
+ }
+
+ //System.out.println(" best w/in group!");
+
+ for (FieldComparator fc : comparators) {
+ fc.copy(bottomSlot, doc);
+ // Necessary because some comparators cache
+ // details of bottom slot; this forces them to
+ // re-cache:
+ fc.setBottom(bottomSlot);
+ }
+ topGroupDoc = doc;
+ }
+ } else {
+ // We're not sure this group will make it into the
+ // queue yet
+ for (int compIDX = 0;; compIDX++) {
+ final int c = reversed[compIDX] * comparators[compIDX].compareBottom(doc);
+ if (c < 0) {
+ // Definitely not competitive -- done
+ //System.out.println(" doc doesn't compete w/ top groups");
+ return;
+ } else if (c > 0) {
+ // Definitely competitive.
+ break;
+ } else if (compIDX == compIDXEnd) {
+ // Ties with bottom, except we know this docID is
+ // > docID in the queue (docs are visited in
+ // order), so not competitive:
+ //System.out.println(" doc doesn't compete w/ top groups");
+ return;
+ }
+ }
+ groupCompetes = true;
+ for (FieldComparator fc : comparators) {
+ fc.copy(bottomSlot, doc);
+ // Necessary because some comparators cache
+ // details of bottom slot; this forces them to
+ // re-cache:
+ fc.setBottom(bottomSlot);
+ }
+ topGroupDoc = doc;
+ //System.out.println(" doc competes w/ top groups");
+ }
+ }
+
+ @Override
+ public boolean acceptsDocsOutOfOrder() {
+ return false;
+ }
+
+ @Override
+ public void setNextReader(AtomicReaderContext readerContext) throws IOException {
+ if (subDocUpto != 0) {
+ processGroup();
+ }
+ subDocUpto = 0;
+ docBase = readerContext.docBase;
+ //System.out.println("setNextReader base=" + docBase + " r=" + readerContext.reader);
+ lastDocPerGroupBits = lastDocPerGroup.getDocIdSet(readerContext).iterator();
+ groupEndDocID = -1;
+
+ currentReaderContext = readerContext;
+ for (int i=0; iThis module enables search result grouping with Lucene, where hits
with the same value in the specified single-valued group field are
-grouped together. For example, if you group by the author
-field, then all documents with the same value in the author
+grouped together. For example, if you group by the author
+field, then all documents with the same value in the author
field fall into a single group.
Grouping requires a number of inputs:
- - groupField: this is the field used for grouping.
- For example, if you use the author field then each
+
-
groupField: this is the field used for grouping.
+ For example, if you use the author field then each
group has all books by the same author. Documents that don't
have this field are grouped under a single group with
- a null group value.
+ a null group value.
- - groupSort: how the groups are sorted. For sorting
+
-
groupSort: how the groups are sorted. For sorting
purposes, each group is "represented" by the highest-sorted
- document according to the groupSort within it. For
+ document according to the groupSort within it. For
example, if you specify "price" (ascending) then the first group
is the one with the lowest price book within it. Or if you
specify relevance group sort, then the first group is the one
containing the highest scoring book.
- - topNGroups: how many top groups to keep. For
+
-
topNGroups: how many top groups to keep. For
example, 10 means the top 10 groups are computed.
- - groupOffset: which "slice" of top groups you want to
+
-
groupOffset: which "slice" of top groups you want to
retrieve. For example, 3 means you'll get 7 groups back
- (assuming topNGroups is 10). This is useful for
+ (assuming topNGroups is 10). This is useful for
paging, where you might show 5 groups per page.
- - withinGroupSort: how the documents within each group
+
-
withinGroupSort: how the documents within each group
are sorted. This can be different from the group sort.
- - maxDocsPerGroup: how many top documents within each
+
-
maxDocsPerGroup: how many top documents within each
group to keep.
- - withinGroupOffset: which "slice" of top
+
-
withinGroupOffset: which "slice" of top
documents you want to retrieve from each group.
-The implementation is two-pass: the first pass ({@link
- org.apache.lucene.search.grouping.FirstPassGroupingCollector})
- gathers the top groups, and the second pass ({@link
- org.apache.lucene.search.grouping.SecondPassGroupingCollector})
- gathers documents within those groups. If the search is costly to
- run you may want to use the {@link
- org.apache.lucene.search.CachingCollector} class, which
- caches hits and can (quickly) replay them for the second pass. This
- way you only run the query once, but you pay a RAM cost to (briefly)
- hold all hits. Results are returned as a {@link
- org.apache.lucene.search.grouping.TopGroups} instance.
+
+There are two grouping implementations here:
+
+ -
+ Arbitrary grouping that can group by any single-valued indexed
+ field, implemented as a two-pass collector: the first pass ({@link
+ org.apache.lucene.search.grouping.FirstPassGroupingCollector})
+ gathers the top groups, and the second pass ({@link
+ org.apache.lucene.search.grouping.SecondPassGroupingCollector})
+ gathers documents within those groups. If the search is costly to
+ run you may want to use the {@link
+ org.apache.lucene.search.CachingCollector} class, which caches
+ hits and can (quickly) replay them for the second pass. This way
+ you only run the query once, but you pay a RAM cost to (briefly)
+ hold all hits. Results are returned as a {@link
+ org.apache.lucene.search.grouping.TopGroups} instance.
+
+ -
+ Indexed groups, using a single pass collector (
DocBlockGroupingCollectorDoc) that
+ is able to group according to the doc blocks created during
+ indexing using IndexWriter's add/updateDocuments API.
+ This is faster (~25% faster QPS) than the generic two-pass
+ collector, but it only works for doc blocks so you must statically
+ commit (during indexing) to which grouping you'll need at search
+ time.
+
+ This implementation does not rely on a single valued grouping
+ field; rather, the blocks in the index define the groups, so your
+ application is free to determine what the grouping criteria is.
+ At search time, you must provide a Filter that marks
+ the last document in each group. This is a substantial memory
+ savings because this collector does not load
+ a DocTermsIndex from the
+ FieldCache.
+
+
+
+The benefit of the arbitrary grouping implementation is you don't have
+to commit at indexing time to a static grouping of your documents.
+But the downside is it's somewhat slower to run, and requires more RAM
+(a FieldCache.DocTermsIndex entry is created).
Known limitations:
- - The group field must be a single-valued indexed field.
- {@link org.apache.lucene.search.FieldCache} is used to load the {@link org.apache.lucene.search.FieldCache.DocTermsIndex} for this field.
+
- For the two-pass grouping collector, the group field must be a
+ single-valued indexed field.
- Unlike Solr's implementation, this module cannot group by
function query values nor by arbitrary queries.
- Sharding is not directly supported, though is not too
@@ -66,7 +96,8 @@
group yourself.
-Typical usage looks like this (using the {@link org.apache.lucene.search.CachingCollector}):
+Typical usage for the generic two-pass collector looks like this
+ (using the {@link org.apache.lucene.search.CachingCollector}):
FirstPassGroupingCollector c1 = new FirstPassGroupingCollector("author", groupSort, groupOffset+topNGroups);
@@ -111,5 +142,50 @@
// Render groupsResult...
+To use the single-pass DocBlockGroupingCollector,
+ first, at indexing time, you must ensure all docs in each group
+ are added as a block, and you have some way to find the last
+ document of each group. One simple way to do this is to add a
+ marker binary field:
+
+
+ // Create Documents from your source:
+ List<Document> oneGroup = ...;
+
+ Field groupEndField = new Field("groupEnd", "x", Field.Store.NO, Field.Index.NOT_ANALYZED);
+ groupEndField.setOmitTermFreqAndPositions(true);
+ groupEndField.setOmitNorms(true);
+ oneGroup.get(oneGroup.size()-1).add(groupEndField);
+
+ // You can also use writer.updateDocuments(); just be sure you
+ // replace an entire previous doc block with this new one. For
+ // example, each group could have a "groupID" field, with the same
+ // value for all docs in this group:
+ writer.addDocuments(oneGroup);
+
+
+Then, at search time, do this up front:
+
+
+ // Set this once in your app & save away for reusing across all queries:
+ Filter groupEndDocs = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("end", "x"))));
+
+
+Finally, do this per search:
+
+
+ // Per search:
+ DocBlockGroupingCollector c = new DocBlockGroupingCollector("author", groupSort, groupOffset+topNGroups, withinGroupSort, docOffset+docsPerGroup, needsScores, groupEndDocs);
+ s.search(new TermQuery(new Term("content", searchTerm)), c);
+ TopGroups groupsResult = c.getTopGroups(groupOffset, docOffset, fillGroupValues, fillFields, getScores, getMaxScores);
+
+ // Render groupsResult...
+
+
+Note that the groupValueo of each OneGroup
+will be null, so if you need to present this value you'll
+have to separately retrieve it (for example using store
+fields, FieldCache, etc.).
+