Index: src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml
===================================================================
--- src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml Sat Sep 12 12:08:36 CEST 2009
+++ src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,462 @@
+
+
+
+
+
+
+
+
+ ${solr.data.dir:./solr/data}
+
+
+
+
+ false
+ 10
+
+
+
+ 32
+ 2147483647
+ 10000
+ 1000
+ 10000
+
+
+ false
+
+
+ org.apache.lucene.index.LogByteSizeMergePolicy
+
+
+ org.apache.lucene.index.ConcurrentMergeScheduler
+
+ 1000
+ 10000
+
+ single
+
+
+
+
+ false
+ 10
+
+ 10
+ 2147483647
+ 10000
+ org.apache.lucene.index.LogDocMergePolicy
+
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 1024
+
+
+
+ true
+
+
+
+
+
+
+ 10
+
+
+
+
+
+
+
+
+
+ true
+
+ collapse
+ facet
+ mlt
+ highlight
+ stats
+ debug
+
+
+
+
+ *:*
+ 0.01
+
+ text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
+
+
+ text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
+
+
+ ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
+
+
+ 3<-1 5<-2 6<90%
+
+ 100
+
+
+
+ *:*
+ 0.01
+
+ text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0
+
+
+ text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5
+
+
+ ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3
+
+
+ 3<-1 5<-2 6<90%
+
+ 100
+
+
+
+ 1000
+ 1.4142135
+ 1 2 foo
+
+
+ sqrt 2
+ log 10
+
+
+
+
+ 4
+ true
+ text,name,subject,title,whitetok
+
+
+
+
+
+
+ 4
+ true
+ text,name,subject,title,whitetok
+
+
+
+
+ false
+
+
+
+ lowerfilt
+
+
+ default
+ lowerfilt
+ spellchecker1
+ true
+
+
+
+ jarowinkler
+ lowerfilt
+
+ org.apache.lucene.search.spell.JaroWinklerDistance
+ spellchecker2
+
+
+
+ solr.FileBasedSpellChecker
+ external
+ spellings.txt
+ UTF-8
+ spellchecker3
+
+
+
+
+
+ termsComp
+
+
+
+
+
+
+ false
+
+ false
+
+ 1
+
+
+ spellcheck
+
+
+
+
+
+
+
+
+
+ tvComponent
+
+
+
+
+
+
+
+ 100
+
+
+
+
+
+ 70
+
+
+
+
+
+
+ ]]>
+ ]]>
+
+
+
+
+
+
+
+
+ max-age=30, public
+
+
+
+
+ solr
+ solrconfig.xml scheam.xml admin-extra.html
+
+
+
+ prefix-${solr.test.sys.prop2}-suffix
+
+
+
+ false
+ true
+ v_t,t_field
+ org.apache.solr.update.processor.TextProfileSignature
+
+
+
+
Index: src/java/org/apache/solr/search/SolrIndexSearcher.java
===================================================================
--- src/java/org/apache/solr/search/SolrIndexSearcher.java (revision 794328)
+++ src/java/org/apache/solr/search/SolrIndexSearcher.java Sat Sep 12 12:08:36 CEST 2009
@@ -17,28 +17,30 @@
package org.apache.solr.search;
-import org.apache.lucene.document.*;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.FieldSelector;
+import org.apache.lucene.document.FieldSelectorResult;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.OpenBitSet;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
+import org.apache.solr.request.UnInvertedField;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.request.UnInvertedField;
-import org.apache.lucene.util.OpenBitSet;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URL;
import java.util.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
@@ -1196,6 +1198,43 @@
}
/**
+ * Returns documents matching both query and the intersection
+ * of filterList, sorted by sort.
+ * Also returns the compete set of documents
+ * matching query and filter
+ * (regardless of offset and len).
+ *
+ * This method is cache aware and may retrieve filter from
+ * the cache or make an insertion into the cache as a result of this call.
+ *
+ * FUTURE: The returned DocList may be retrieved from a cache.
+ *
+ * The DocList and DocSet returned should not be modified.
+ *
+ * @param query
+ * @param filterList may be null
+ * @param docSet filter docSet
+ * @param lsort criteria by which to sort (if null, query relevance is used)
+ * @param offset offset into the list of documents to return
+ * @param len maximum number of documents to return
+ * @param flags user supplied flags for the result set
+ * @return DocListAndSet meeting the specified criteria, should not be modified by the caller.
+ * @throws IOException
+ */
+ public DocListAndSet getDocListAndSet(Query query, List filterList, DocSet docSet, Sort lsort, int offset, int len, int flags) throws IOException {
+ //DocListAndSet ret = new DocListAndSet();
+ //getDocListC(ret,query,filterList,docSet,lsort,offset,len, flags |= GET_DOCSET);
+
+ QueryCommand qc = new QueryCommand();
+ qc.setQuery(query).setFilterList(filterList).setFilter(docSet);
+ qc.setSort(lsort).setOffset(offset).setLen(len).setFlags(flags |= GET_DOCSET);
+ QueryResult result = new QueryResult();
+ getDocListC(result,qc);
+
+ return result.getDocListAndSet();
+ }
+
+ /**
* Returns documents matching both query and filter
* and sorted by sort. Also returns the compete set of documents
* matching query and filter (regardless of offset and len).
Index: src/java/org/apache/solr/handler/component/CollapseComponent.java
===================================================================
--- src/java/org/apache/solr/handler/component/CollapseComponent.java Sat Sep 12 17:20:30 CEST 2009
+++ src/java/org/apache/solr/handler/component/CollapseComponent.java Sat Sep 12 17:20:30 CEST 2009
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.common.params.CollapseParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.search.*;
+
+import java.io.IOException;
+import java.net.URL;
+
+/**
+ * Collapse component is responsible for do field collapsing with an implementation of {@link org.apache.solr.search.DocumentCollapser},
+ * that does the most of the work. Collapsing is activated by specifying collapse.field in the request
+ * as parameter.
+ * If this parameter is not specified it falls back to the {@link QueryComponent#process(ResponseBuilder)} method.
+ * collapse.facet with value after is specified,
+ * it replaces the collapsed docSet from DocListAndSet with the uncollapsed docset, so that facetation
+ * counts are based on the uncollapsed search result (which is usually desired behaviour)
+ *
+ * @version $Id: QueryComponent.java 602341 2007-12-08 07:27:49Z ryan $
+ * @since solr 1.3
+ */
+public class CollapseComponent extends QueryComponent {
+
+ /**
+ * Parses some parameters from the request in preparation for the collapsing when collapsing is enabled otherwise
+ * behaves like {@link super#prepare(ResponseBuilder)}.
+ *
+ * @param rb The response builder of the current search request
+ * @throws IOException When index search problems occur
+ */
+ @Override
+ public void prepare(ResponseBuilder rb) throws IOException {
+ super.prepare(rb);
+ rb.req.getContext().put("collapseRequest", resolveCollapseRequest(rb));
+ }
+
+ /**
+ * Actually run the query
+ */
+ @Override
+ public void process(ResponseBuilder rb) throws IOException {
+ CollapseRequest collapseRequest = (CollapseRequest) rb.req.getContext().remove("collapseRequest");
+ if (collapseRequest == null) {
+ super.process(rb);
+ return;
+ }
+ doProcess(rb, collapseRequest);
+ }
+
+
+ // ================================================= Helpers =========================================================
+
+ protected CollapseRequest resolveCollapseRequest(ResponseBuilder rb) throws IOException {
+ SolrParams params = rb.req.getParams();
+ if (params.get(CollapseParams.COLLAPSE_FIELD) == null) {
+ return null;
+ }
+
+ String facet = params.get(CollapseParams.COLLAPSE_FACET);
+ CollapseParams.CollapseFacet collapseFacet = (facet != null) ? CollapseParams.CollapseFacet.get(facet) : CollapseParams.CollapseFacet.AFTER;
+
+ String type = params.get(CollapseParams.COLLAPSE_TYPE);
+ CollapseParams.CollapseType collapseType = (type != null) ? CollapseParams.CollapseType.get(type) : CollapseParams.CollapseType.NORMAL;
+ DocumentCollapser documentCollapser = constructCollapser(rb, collapseType);
+ return new CollapseRequest(documentCollapser, collapseFacet);
+ }
+
+ /**
+ * Executes a search and collapses the search results.
+ *
+ * @param rb The response builder of the current search request
+ * @param collapseRequest
+ * @throws IOException When index search problems occur
+ */
+ protected void doProcess(ResponseBuilder rb, CollapseComponent.CollapseRequest collapseRequest) throws IOException {
+ SolrQueryResponse rsp = rb.rsp;
+ SolrQueryRequest req = rb.req;
+ SolrIndexSearcher searcher = req.getSearcher();
+ DocumentCollapseResult collapseResult = collapseRequest.getCollapser().collapse(rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort());
+ DocListAndSet results = searcher.getDocListAndSet(rb.getQuery(),
+ collapseResult == null ? rb.getFilters() : null,
+ collapseResult.getCollapsedDocset(),
+ rb.getSortSpec().getSort(),
+ rb.getSortSpec().getOffset(),
+ rb.getSortSpec().getCount(),
+ rb.getFieldFlags());
+
+ //for getting the facet count BEFORE the collapsing, we must
+ //get the doc. collection without filtering by the collapseResult.
+ boolean facetAfterCollapse = (collapseRequest.getCollapseFacet() == CollapseParams.CollapseFacet.AFTER);
+ if (!facetAfterCollapse) {
+ results.docSet = collapseResult.getUnCollapsedDocset();
+ }
+
+ rb.setResults(results);
+ rsp.add("collapse_counts", collapseRequest.getCollapser().getCollapseInfo(searcher, results.docList));
+ rsp.add("response", results.docList);
+ }
+
+ /**
+ * Returns the field collapser implementation based on the specified collapseType
+ *
+ * @param rb The response builder of the current search request
+ * @param collapseType The type of collapsing for the current search
+ * @return collapser implementation
+ * @throws IOException When index search problems occur
+ */
+ protected DocumentCollapser constructCollapser(ResponseBuilder rb, CollapseParams.CollapseType collapseType) throws IOException {
+ switch (collapseType) {
+ case ADJACENT:
+ return new AdjacentDocumentCollapser(rb);
+ case NORMAL:
+ return new NonAdjacentDocumentCollapser(rb);
+ default:
+ throw new UnsupportedOperationException("Method not implemented");
+ }
+ }
+
+ /////////////////////////////////////////////
+ /// SolrInfoMBean
+ ////////////////////////////////////////////
+
+ @Override
+ public String getDescription() {
+ return "Field Collapsing";
+ }
+
+ @Override
+ public String getVersion() {
+ return "";
+ }
+
+ @Override
+ public String getSourceId() {
+ return "";
+ }
+
+ @Override
+ public String getSource() {
+ return "";
+ }
+
+ @Override
+ public URL[] getDocs() {
+ return null;
+ }
+
+
+ // ============================================ Inner Classes ========================================================
+
+ class CollapseRequest {
+
+ private final DocumentCollapser collapser;
+ private final CollapseParams.CollapseFacet collapseFacet;
+
+ private CollapseRequest(DocumentCollapser collapser, CollapseParams.CollapseFacet collapseFacet) {
+ this.collapser = collapser;
+ this.collapseFacet = collapseFacet;
+ }
+
+ public DocumentCollapser getCollapser() {
+ return collapser;
+ }
+
+ public CollapseParams.CollapseFacet getCollapseFacet() {
+ return collapseFacet;
+ }
+ }
+}
Index: src/java/org/apache/solr/search/AdjacentDocumentCollapser.java
===================================================================
--- src/java/org/apache/solr/search/AdjacentDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/search/AdjacentDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.FieldCache;
+import org.apache.solr.handler.component.ResponseBuilder;
+
+import java.io.IOException;
+
+/**
+ * Adjacent collapsing behaviour. Has the same behaviour as {@link NonAdjacentDocumentCollapser} but with one extra constraint.
+ * The collapsing happens in an adjacent manner. Meaning that only documents that are next to each other in a continuous
+ * row and that have the same field value might be collapsed.
+ */
+public class AdjacentDocumentCollapser extends AbstractDocumentCollapser {
+
+ /**
+ * See {@link super#AbstractDocumentCollapser(org.apache.solr.handler.component.ResponseBuilder)}
+ */
+ public AdjacentDocumentCollapser(ResponseBuilder rb) throws IOException {
+ super(rb);
+ }
+
+ // ================================================= Helpers =========================================================
+
+ /**
+ * Applies adjacent collapsing on the specified uncollapsedDocSet.
+ *
+ * @param uncollapsedDocset The uncollapsed docset
+ * @param values The fieldvalues the collapse on
+ */
+ protected void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values) {
+ int docCount = 0; // how many documents we have processed
+ int repeatCount = 0; // how many times we have seen the same value in a
+ int collapseCount = 0; // how many documents we have collapsed in this
+ int collapseId = -1; // the document we're collapsing into
+ String collapseValue = null;
+
+
+ long startTime = System.currentTimeMillis();
+ for (DocIterator i = uncollapsedDocset.iterator(); i.hasNext();) {
+ int currentId = i.nextDoc();
+ String currentValue = values.lookup [values.order[currentId]];
+
+ // Initializing
+ if (collapseValue == null) {
+ repeatCount = 0;
+ collapseCount = 0;
+ collapseId = currentId;
+ collapseValue = currentValue;
+
+ // Collapse the document if the field value is the same and
+ // we have a run of at least collapseThreshold uncollapsedDocset.
+ } else if (collapseValue.equals(currentValue)) {
+ if (++repeatCount >= collapseTreshold) {
+ collapseCount++;
+ addCollapsedDoc(currentId, currentValue);
+ } else {
+ addDoc(currentId);
+ }
+ } else {
+ addDoc(collapseId);
+ if (collapseCount > 0) {
+ collapseCounts.put(collapseId, collapseCount);
+ }
+
+ repeatCount = 0;
+ collapseCount = 0;
+ collapseId = currentId;
+ collapseValue = currentValue;
+ }
+
+ // Stop after collapseMaxDocs documents
+ if (++docCount >= collapseMaxDocs) {
+ break;
+ }
+ }
+
+ if (collapseId != -1) {
+ addDoc(collapseId);
+ }
+
+ if (collapseCount > 0) {
+ collapseCounts.put(collapseId, collapseCount);
+ }
+
+ timeCollapsing = System.currentTimeMillis() - startTime;
+ }
+
+}
Index: src/solrj/org/apache/solr/client/solrj/SolrQuery.java
===================================================================
--- src/solrj/org/apache/solr/client/solrj/SolrQuery.java (revision 749056)
+++ src/solrj/org/apache/solr/client/solrj/SolrQuery.java Sat Sep 12 12:08:36 CEST 2009
@@ -17,11 +17,7 @@
package org.apache.solr.client.solrj;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.FacetParams;
-import org.apache.solr.common.params.HighlightParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.StatsParams;
+import org.apache.solr.common.params.*;
/**
@@ -571,6 +567,42 @@
return this.getInt(CommonParams.TIME_ALLOWED);
}
+ /**
+ * Enables field collapsing for the current query with the specified field.
+ *
+ * @param field The field to collapse on
+ * @return the updated SolrQuery
+ */
+ public SolrQuery enableFieldCollapsing(String field) {
+ add(CollapseParams.COLLAPSE_FIELD, field);
+ return this;
+ }
+
+ /**
+ * Enables the inclusion of collapsed documents in the response. The fields parameter specify what fields are returned
+ * from the collasped documents. The fewer fields to return the better the performance is. When the argument specified
+ * is null or empty all fields will be returned.
+ *
+ * @param fields The fields to return for collapsed documents. If null or empty all fields are returned.
+ * @return the updated SolrQuery
+ */
+ public SolrQuery includeCollapsedDocuments(String... fields) {
+ add(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS, "true");
+ if (fields == null || fields.length < 1) {
+ return this;
+ }
+
+ StringBuilder fl = new StringBuilder();
+ for (int i = 0; i < fields.length; i++) {
+ fl.append(fields[i]);
+ if (i + 1 < fields.length) {
+ fl.append(',');
+ }
+ }
+ add(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, fl.toString());
+ return this;
+ }
+
///////////////////////
// Utility functions
///////////////////////
Index: src/test/org/apache/solr/handler/component/CollapseComponentTest.java
===================================================================
--- src/test/org/apache/solr/handler/component/CollapseComponentTest.java Sat Sep 12 12:08:36 CEST 2009
+++ src/test/org/apache/solr/handler/component/CollapseComponentTest.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.component;
+
+import org.apache.solr.common.params.CollapseParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.search.AdjacentDocumentCollapser;
+import org.apache.solr.search.NonAdjacentDocumentCollapser;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Tests for {@link org.apache.solr.handler.component.CollapseComponent}.
+ */
+public class CollapseComponentTest extends AbstractSolrTestCase {
+
+ public String getSchemaFile() {
+ return "schema-fieldcollapse.xml";
+ }
+
+ public String getSolrConfigFile() {
+ return "solrconfig-fieldcollapse.xml";
+ }
+
+ public void testResolveCollpaseRequest_AdjacentCollapserFacetsBefore() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()});
+ parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.ADJACENT.toString()});
+ responseBuilder.req.setParams(new ModifiableSolrParams(parameters));
+
+ CollapseComponent collapseComponent = new CollapseComponent();
+ CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder);
+ assertEquals(CollapseParams.CollapseFacet.BEFORE, request.getCollapseFacet());
+ assertEquals(AdjacentDocumentCollapser.class, request.getCollapser().getClass());
+ }
+
+ public void testResolveCollpaseRequest_NonAdjacentCollapserFacetsAfter() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.AFTER.toString()});
+ parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.NORMAL.toString()});
+ responseBuilder.req.setParams(new ModifiableSolrParams(parameters));
+
+ CollapseComponent collapseComponent = new CollapseComponent();
+ CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder);
+ assertEquals(CollapseParams.CollapseFacet.AFTER, request.getCollapseFacet());
+ assertEquals(NonAdjacentDocumentCollapser.class, request.getCollapser().getClass());
+ }
+
+ public void testResolveCollpaseRequest_requestIsNull() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()});
+ parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.ADJACENT.toString()});
+ responseBuilder.req.setParams(new ModifiableSolrParams(parameters));
+
+ CollapseComponent collapseComponent = new CollapseComponent();
+ CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder);
+ assertNull(request);
+ }
+
+}
Index: src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java
===================================================================
--- src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java (revision 763791)
+++ src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java Sat Sep 12 12:08:36 CEST 2009
@@ -17,18 +17,13 @@
package org.apache.solr.client.solrj.response;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.util.NamedList;
import org.apache.solr.client.solrj.SolrServer;
import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+import java.util.*;
+
/**
*
* @version $Id: QueryResponse.java 763791 2009-04-09 20:24:34Z ryan $
@@ -46,6 +41,7 @@
private NamedList _highlightingInfo = null;
private NamedList _spellInfo = null;
private NamedList _statsInfo = null;
+ private NamedList _collapseInfo = null;
// Facet stuff
private Map _facetQuery = null;
@@ -59,6 +55,9 @@
// SpellCheck Response
private SpellCheckResponse _spellResponse = null;
+ // Field collapse response
+ private FieldCollapseResponse _fieldCollapseResponse = null;
+
// Field stats Response
private Map _fieldStatsInfo = null;
@@ -118,9 +117,17 @@
_statsInfo = (NamedList) res.getVal( i );
extractStatsInfo( _statsInfo );
}
+ else if ("collapse_counts".equals(n)) {
+ _collapseInfo = (NamedList) res.getVal(i);
+ extractFieldCollapseInfo(_collapseInfo);
- }
- }
+ }
+ }
+ }
+ private void extractFieldCollapseInfo(NamedList collapseInfo) {
+ _fieldCollapseResponse = new FieldCollapseResponse(collapseInfo);
+ }
+
private void extractSpellCheckInfo(NamedList spellInfo) {
_spellResponse = new SpellCheckResponse(spellInfo);
}
@@ -274,6 +281,10 @@
return _spellResponse;
}
+ public FieldCollapseResponse getFieldCollapseResponse() {
+ return _fieldCollapseResponse;
+ }
+
/**
* See also: {@link #getLimitingFacets()}
*/
Index: src/common/org/apache/solr/common/params/CollapseParams.java
===================================================================
--- src/common/org/apache/solr/common/params/CollapseParams.java Sat Sep 12 12:08:36 CEST 2009
+++ src/common/org/apache/solr/common/params/CollapseParams.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,96 @@
+package org.apache.solr.common.params;
+
+import org.apache.solr.common.SolrException;
+
+public interface CollapseParams {
+
+ /**
+ * The field to collapse results on.
+ */
+ public static final String COLLAPSE_FIELD = "collapse.field";
+
+ /**
+ * Type of collapsing to perform: "normal" or "adjacent".
+ */
+ public static final String COLLAPSE_TYPE = "collapse.type";
+
+ public enum CollapseType {
+ NORMAL, ADJACENT;
+
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ public static CollapseType get(String label) {
+ try {
+ return valueOf(label.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, label
+ + " is not a valid type of field collapsing", e);
+ }
+ }
+ }
+
+ /**
+ * Apply faceting before or after collapsing.
+ */
+ public static final String COLLAPSE_FACET = "collapse.facet";
+
+ public enum CollapseFacet {
+ BEFORE, AFTER;
+
+ public String toString() {
+ return super.toString().toLowerCase();
+ }
+
+ public static CollapseFacet get(String label) {
+ try {
+ return valueOf(label.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, label
+ + " is not a valid faceting mode for field collapsing", e);
+ }
+ }
+ }
+
+ /**
+ * The number of documents with the same value for collapse.field after which
+ * collapsing kicks in.
+ */
+ public static final String COLLAPSE_THRESHOLD = "collapse.threshold";
+
+ /**
+ * @deprecated Deprecated in favour of collapse.threshold.
+ */
+ public static final String COLLAPSE_MAX = "collapse.max";
+
+ /**
+ * Maximum number of documents to process during field collapsing.
+ */
+ public static final String COLLAPSE_MAXDOCS = "collapse.maxdocs";
+
+ /**
+ * Return collapse count for each document? Defaults to true.
+ */
+ public static final String COLLAPSE_INFO_DOC = "collapse.info.doc";
+
+ /**
+ * Return collapse count for each field value? Defaults to true.
+ */
+ public static final String COLLAPSE_INFO_COUNT = "collapse.info.count";
+
+ /**
+ * Parameter indicating to return the collapsed documents in the response
+ */
+ public static final String COLLAPSE_INCLUDE_COLLAPSED_DOCS = "collapse.includeCollapsedDocs";
+
+ /**
+ * Parameter indicating which fields to return from collapsed documents in the response.
+ */
+ public static final String COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS = "collapse.includeCollapsedDocs.fl";
+
+ /**
+ * Parameter indicating wheter to include collapse debug information
+ */
+ public static final String COLLAPSE_DEBUG = "collapse.debug";
+}
Index: src/java/org/apache/solr/search/AbstractDocumentCollapser.java
===================================================================
--- src/java/org/apache/solr/search/AbstractDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/search/AbstractDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,480 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CollapseParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.update.DocumentBuilder;
+
+import java.io.IOException;
+import java.util.*;
+
+/**
+ * Base Collapser implementation
+ */
+public abstract class AbstractDocumentCollapser implements DocumentCollapser {
+
+ /**
+ * Field to use to collapse results. Parameter.
+ */
+ protected final String collapseField;
+
+ /**
+ * Number of documents with the same value for collapseField after which
+ * collapsing kicks in. Parameter.
+ */
+ protected final int collapseTreshold;
+
+
+ /** Collapse State *********************************************** */
+
+
+ /**
+ * Maximum number of documents to process during field collapsing.
+ * Parameter.
+ */
+ protected final int collapseMaxDocs;
+
+ /**
+ * Whether to include collapse count for each document id in the response. Parameter.
+ */
+ protected final boolean collapseInfoDoc;
+
+ /**
+ * Whether to include collapse count for each field value in the response. Parameter.
+ */
+ protected final boolean collapseInfoCount;
+
+ protected final boolean includeCollapsedDocuments;
+
+ protected final Set includeCollapsedDocumentsFields;
+
+ /**
+ * Whether to include debug information in the request
+ */
+ protected boolean includeDebugInformation;
+
+ /**
+ * Number of documents that have been collapsed into the key document.
+ */
+ protected final Map collapseCounts;
+
+ /**
+ * Maxmimum size for a HashDocSet.
+ */
+ protected final int hashMaxSize;
+
+ /**
+ * Buffer for collecting documents. Gets turned into different types of
+ * DocSet depending on the number of documents we end up with.
+ */
+ protected int[] docbuf;
+
+ /**
+ * Number of documents in docbuf.
+ */
+ protected int docbufSize = 0;
+
+ /**
+ * Maximum document id currently in docbuf. Only valid while docbufSize <
+ * hashMaxSize.
+ */
+ protected int docbufMaxDoc = 0;
+
+ /**
+ * Bitset representation of docbuf. Gets created when docbufSize >=
+ * hashMaxSize.
+ */
+ protected OpenBitSet docbufBitSet;
+
+ /**
+ * The result of executing the query and filter queries, without collapsing.
+ */
+ protected DocSet uncollapsedDocSet;
+
+ protected Map collapsedDocuments;
+
+ protected FieldCache.StringIndex fieldValues;
+ protected final SolrIndexSearcher searcher;
+
+
+ protected final int flags;
+ /**
+ * Debug Information ********************************************
+ */
+
+ protected long timeCollapsing = 0;
+ protected long timeConvertToBitSet = 0;
+ protected long timeCreateDocSet = 0;
+ protected long timeCreateCollapseInfo = 0;
+ protected long timeCreateUncollapedDocset = 0;
+ protected String debugDocSetInfo = "unknown";
+
+
+ // ============================================ Constructors =======================================================
+
+ /**
+ * Constructs a base collapser.
+ *
+ * @param rb The response builder
+ * @throws IOException if index searcher related problems occur
+ */
+ protected AbstractDocumentCollapser(ResponseBuilder rb) throws IOException {
+ // Allocate data structures
+ hashMaxSize = rb.req.getCore().getSolrConfig().hashDocSetMaxSize;
+ docbuf = new int[hashMaxSize];
+ collapseCounts = new HashMap();
+ collapsedDocuments = new HashMap();
+ this.searcher = rb.req.getSearcher();
+ this.flags = rb.getFieldFlags();
+
+ // parsing Solr parameters
+ SolrParams params = rb.req.getParams();
+ collapseField = params.required().get(CollapseParams.COLLAPSE_FIELD);
+ Integer ct = params.getInt(CollapseParams.COLLAPSE_THRESHOLD);
+ if (ct == null) {
+ ct = params.getInt(CollapseParams.COLLAPSE_MAX);
+ }
+ collapseTreshold = (ct != null) ? ct : 1;
+ checkCollapseField(rb.req.getSchema());
+
+ int collapseMaxDocs = params.getInt(CollapseParams.COLLAPSE_MAXDOCS, 0);
+ if (collapseMaxDocs <= 0) {
+ this.collapseMaxDocs = searcher.maxDoc();
+ } else {
+ this.collapseMaxDocs = collapseMaxDocs;
+ }
+
+ collapseInfoDoc = params.getBool(CollapseParams.COLLAPSE_INFO_DOC, true);
+ collapseInfoCount = params.getBool(CollapseParams.COLLAPSE_INFO_COUNT, true);
+ includeDebugInformation = params.getBool(CollapseParams.COLLAPSE_DEBUG ,false);
+
+ // parsing parameters for inclusion of collapsed documents
+ includeCollapsedDocuments = params.getBool(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS, false);
+ String collapsedDocumentFields = params.get(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS);
+ if (collapsedDocumentFields != null && collapsedDocumentFields.length() > 0) {
+ includeCollapsedDocumentsFields = new HashSet(Arrays.asList(collapsedDocumentFields.split(",")));
+ } else {
+ includeCollapsedDocumentsFields = null;
+ }
+ }
+
+
+ // ======================================= Public API methods ========================================================
+
+ /**
+ * {@inheritDoc}
+ */
+ public DocumentCollapseResult collapse(Query mainQuery, List filterQueries, Sort sort) throws IOException {
+ long startTime = System.currentTimeMillis();
+ doQuery(mainQuery, filterQueries, sort);
+ timeCreateUncollapedDocset = System.currentTimeMillis() - startTime;
+ fieldValues = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), collapseField);
+ doCollapsing(uncollapsedDocSet, fieldValues);
+ return createDocumentCollapseResult();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public NamedList getCollapseInfo(SolrIndexSearcher searcher, DocList docs) throws IOException {
+ if (uncollapsedDocSet == null) {
+ throw new IllegalStateException("Invoke the collapse method before invoking getCollapseInfo method");
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ NamedList result = new NamedList();
+ result.add("field", collapseField);
+
+ IndexSchema schema = searcher.getSchema();
+ FieldType collapseFieldType = schema.getField(collapseField).getType();
+ SchemaField uniqueKeyField = schema.getUniqueKeyField();
+ String uniqueKeyName = (uniqueKeyField != null) ? uniqueKeyField.getName() : null;
+
+ if (collapseInfoDoc || collapseInfoCount) {
+ NamedList resDoc = null;
+ NamedList resCount = null;
+ NamedList collapsedDocs = null;
+ DocumentBuilder documentBuilder = null;
+ IndexReader reader = null;
+
+ if (collapseInfoDoc) {
+ resDoc = new NamedList();
+ result.add("doc", resDoc);
+ reader = searcher.getReader();
+ }
+
+ if (collapseInfoCount) {
+ resCount = new NamedList();
+ result.add("count", resCount);
+ }
+
+ if (includeCollapsedDocuments) {
+ collapsedDocs = new NamedList();
+ result.add("collapsedDocs", collapsedDocs);
+ documentBuilder = new DocumentBuilder(schema);
+ }
+
+ for (DocIterator i = docs.iterator(); i.hasNext();) {
+ int id = i.nextDoc();
+ Integer count = collapseCounts.get(id);
+ if (count != null) {
+ if (collapseInfoDoc && uniqueKeyName != null) {
+ resDoc.add(reader.document(id).get(uniqueKeyName), count);
+ }
+
+ if (collapseInfoCount) {
+ String value = fieldValues.lookup[fieldValues.order[id]];
+ resCount.add(collapseFieldType.indexedToReadable(value), count);
+ }
+
+ if (includeCollapsedDocuments) {
+ String value = fieldValues.lookup[fieldValues.order[id]];
+ OpenBitSet bitSet = collapsedDocuments.get(value);
+ if (bitSet == null || bitSet.isEmpty()) {
+ continue;
+ }
+
+ SolrDocumentList documentList = new SolrDocumentList();
+ for (DocIdSetIterator iterator = bitSet.iterator(); iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;) {
+ Document luceneDocument = searcher.doc(iterator.docID(), includeCollapsedDocumentsFields);
+
+ // the luceneDocument may still contain fields that are not specified in the includeCollapsedDocumentsFields
+ // this is because the document was read from the document cache. Offcourse we only want to return requested the fields.
+ // There is also another doc() method that accepts a FieldSelector instead of a list of fields, but that
+ // method does not use the Solr document cache, which from my perspective is a necessity
+ if (includeCollapsedDocumentsFields != null && luceneDocument.getFields().size() > includeCollapsedDocumentsFields.size()) {
+ Document document = new Document();
+ for (Object object : luceneDocument.getFields()) {
+ Fieldable field = (Fieldable) object;
+ if (includeCollapsedDocumentsFields.contains(field.name())) {
+ document.add(field);
+ }
+ }
+ luceneDocument = document;
+ }
+
+ SolrDocument solrDocument = new SolrDocument();
+ documentBuilder.loadStoredFields(solrDocument, luceneDocument);
+ documentList.add(solrDocument);
+ }
+ documentList.setNumFound(bitSet.cardinality());
+ collapsedDocs.add(value, documentList);
+ }
+ }
+ }
+ }
+
+ if (includeDebugInformation) {
+ result.add("debug", getDebugInfo());
+ }
+
+ timeCreateCollapseInfo = System.currentTimeMillis() - startTime;
+ return result;
+ }
+
+
+ // ================================================= Helpers =======================================================
+
+ /**
+ * Executes implementation specific collapsing algorithm.
+ *
+ * @param uncollapsedDocset The uncollapsed DocSet
+ * @param values The field values
+ */
+ protected abstract void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values);
+
+ /**
+ * @return timing information for field collapsing process.
+ */
+ protected NamedList getDebugInfo() {
+ long totalTime = timeCreateUncollapedDocset + timeCollapsing + timeCreateCollapseInfo + timeConvertToBitSet + timeCreateDocSet;
+ NamedList namedList = new NamedList();
+ namedList.add("Docset type", debugDocSetInfo);
+ namedList.add("Total collapsing time(ms)", totalTime);
+ namedList.add("Create uncollapsed docset(ms)", timeCreateUncollapedDocset);
+ namedList.add(String.format("%s collapsing time(ms)", getClass().getSimpleName()), timeCollapsing);
+ namedList.add("Creating collapseinfo time(ms)", timeCreateCollapseInfo);
+ namedList.add("Convert to bitset time(ms)", timeConvertToBitSet);
+ namedList.add("Create collapsed docset time(ms)", timeCreateDocSet);
+ return namedList;
+ }
+
+ /**
+ * Creates the document collapse result based on the uncollapsed docset and the collapsed docset.
+ *
+ * @return he document collapse result
+ */
+ protected DocumentCollapseResult createDocumentCollapseResult() {
+ long startTime = System.currentTimeMillis();
+ DocSet result = (docbufBitSet != null) ? new BitDocSet(docbufBitSet) : new HashDocSet(docbuf, 0, docbufSize);
+ timeCreateDocSet = System.currentTimeMillis() - startTime;
+ debugDocSetInfo = result.getClass().getSimpleName() + "(" + docbufSize + ")";
+ return new DocumentCollapseResult(result, uncollapsedDocSet);
+ }
+
+ /**
+ * Adds a document to the internal document buffer.
+ *
+ * @param doc The lucene identifier of the document to add
+ */
+ protected void addDoc(int doc) {
+ // If we have less than hashMaxSize documents, just
+ // keep adding them to docbuf. We will turn them into
+ // a HashDocSet later.
+
+ if (docbufSize < hashMaxSize) {
+ docbuf[docbufSize] = doc;
+ if (doc > docbufMaxDoc) {
+ docbufMaxDoc = doc;
+ }
+ } else {
+ // We have exceeded hashMaxSize. Allocate a bit set
+ // if we don't have one yet, then add to that.
+ if (docbufBitSet == null) {
+ long startTime = System.currentTimeMillis();
+ docbufBitSet = new OpenBitSet(docbufMaxDoc + 1);
+ for (int i = 0; i < docbufSize; i++) {
+ docbufBitSet.fastSet(docbuf[i]);
+ }
+ timeConvertToBitSet = System.currentTimeMillis() - startTime;
+ }
+ docbufBitSet.set(doc);
+ }
+ docbufSize++;
+ }
+
+ /**
+ * Adds a collapsed document to a set of other collapsed documents with a common field value.
+ *
+ * @param doc The lucene identifier of the document
+ * @param fieldValue The field value this document is collapsed under
+ */
+ protected void addCollapsedDoc(int doc, String fieldValue) {
+ if (!includeCollapsedDocuments) {
+ return;
+ }
+
+ OpenBitSet bitSet = collapsedDocuments.get(fieldValue);
+ if (bitSet == null) {
+ collapsedDocuments.put(fieldValue, bitSet = new OpenBitSet());
+ }
+ // We cannot use fastset because we do not know the size upfront.
+ // The set method is more expensive than fastset, but that is the best for now.
+ // Setting bitSet to maxDoc may increase the memory footprint dramatically, because for each distinct fieldvalue
+ // there is a bitset
+ bitSet.set(doc);
+ }
+
+ protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException {
+ uncollapsedDocSet = searcher.getDocList(mainQuery, filterQueries, sort, 0, collapseMaxDocs, flags);
+ }
+
+ /**
+ * Method that checks if proper field collapsing is actually possible with the current collapse field.
+ * If the collapse field does not meet the proper requirements a runtime exception is thrown.
+ * A runtime exception is thrown under the following circumstances:
+ *
+ * When the collapse field does not exists in the schema
+ * When the collapse field is multivalued in the schema
+ * When the collapse field is tokenized in the schema
+ *
+ *
+ * For example when a field is tokenized, only the last token of the field can be retrieved from the fieldcache. This
+ * results in field collapsing only on the last token of a field value instead of the complete field value.
+ *
+ * When the field values from a multivalued field are returned from the FieldCache then an exception may
+ * be thrown. This happens when there are more terms in a field than documents.
+ *
+ * @param schema The index schema
+ */
+ protected void checkCollapseField(IndexSchema schema) {
+ SchemaField schemaField = schema.getFieldOrNull(collapseField);
+ if (schemaField == null) {
+ throw new RuntimeException("Could not collapse, because collapse field does not exist in the schema.");
+ }
+
+ if (schemaField.multiValued()) {
+ throw new RuntimeException("Could not collapse, because collapse field is multivalued");
+ }
+
+ if (schemaField.getType().isTokenized()) {
+ throw new RuntimeException("Could not collapse, because collapse field is tokenized");
+ }
+ }
+
+ // =========================================== Getters / Setters ===================================================
+
+ /**
+ * @return field to use to collapse results
+ */
+ public String getCollapseField() {
+ return collapseField;
+ }
+
+ /**
+ * @return whether to include collapse count for each field value in the response
+ */
+ public boolean isCollapseInfoCount() {
+ return collapseInfoCount;
+ }
+
+ /**
+ * @return whether to unclude collapse counts for each document in the response
+ */
+ public boolean isCollapseInfoDoc() {
+ return collapseInfoDoc;
+ }
+
+ /**
+ * @return maximum number of documents to process during field collapsing
+ */
+ public int getCollapseMaxDocs() {
+ return collapseMaxDocs;
+ }
+
+ /**
+ * @return number of documents with the same value for collapseField after which collapsing kicks in.
+ */
+ public int getCollapseTreshold() {
+ return collapseTreshold;
+ }
+
+ /**
+ * @return number of documents that have been collapsed into the key document
+ */
+ public Map getCollapseCounts() {
+ return Collections.unmodifiableMap(collapseCounts);
+ }
+}
Index: src/test/test-files/solr/conf/schema-fieldcollapse.xml
===================================================================
--- src/test/test-files/solr/conf/schema-fieldcollapse.xml Sat Sep 12 12:08:36 CEST 2009
+++ src/test/test-files/solr/conf/schema-fieldcollapse.xml Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,164 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ title
+ id
+
+
Index: src/test/org/apache/solr/search/AdjacentCollapserTest.java
===================================================================
--- src/test/org/apache/solr/search/AdjacentCollapserTest.java Sat Sep 12 12:08:36 CEST 2009
+++ src/test/org/apache/solr/search/AdjacentCollapserTest.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.FieldCache;
+import org.apache.solr.common.params.CollapseParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Unit tests for {@link AdjacentDocumentCollapser}.
+ */
+public class AdjacentCollapserTest extends AbstractSolrTestCase {
+
+ private AdjacentDocumentCollapser collapseFilter;
+
+ public String getSchemaFile() {
+ return "schema-fieldcollapse.xml";
+ }
+
+ public String getSolrConfigFile() {
+ return "solrconfig-fieldcollapse.xml";
+ }
+
+ /*
+ * Pass condition: document with id 2 and 6 are collapsed
+ */
+ public void testAdjacentCollapse() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map params = new HashMap();
+ params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"});
+ responseBuilder.req.setParams(new ModifiableSolrParams(params));
+
+ collapseFilter = new AdjacentDocumentCollapser(responseBuilder);
+ // adjacent collapsing can only works with an order docset
+ String[] values = new String[]{"a", "b", "c", "z"};
+ int[] order = new int[]{3, 0, 0, 1, 2, 1, 1};
+ FieldCache.StringIndex index = new FieldCache.StringIndex(order, values);
+ int[] docIds = new int[]{0, 1, 2, 3, 4, 5, 6};
+ DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7);
+ collapseFilter.doCollapsing(uncollapsedDocset, index);
+
+ Map collapseCounts = collapseFilter.getCollapseCounts();
+ assertEquals(2, collapseCounts.size());
+ assertEquals(1, (int) collapseCounts.get(1));
+ assertEquals(1, (int) collapseCounts.get(5));
+
+ DocSet docHeadsDocSet = collapseFilter.createDocumentCollapseResult().getCollapsedDocset();
+ assertEquals(5, docHeadsDocSet.size());
+
+ boolean zeroFound = false;
+ boolean oneFound = false;
+ boolean twoFound = false;
+ boolean threeFound = false;
+ boolean fourFound = false;
+ boolean fiveFound = false;
+ boolean sixFound = false;
+ for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) {
+ int docId = i.nextDoc();
+ if (docId == 0) {
+ zeroFound = true;
+ } else if (docId == 1) {
+ oneFound = true;
+ } else if (docId == 2) {
+ twoFound = true;
+ } else if (docId == 3) {
+ threeFound = true;
+ } else if (docId == 4) {
+ fourFound = true;
+ } else if (docId == 5) {
+ fiveFound = true;
+ } else if (docId == 6) {
+ sixFound = true;
+ }
+ }
+
+ assertTrue(zeroFound);
+ assertTrue(oneFound);
+ assertFalse(twoFound);
+ assertTrue(threeFound);
+ assertTrue(fourFound);
+ assertTrue(fiveFound);
+ assertFalse(sixFound);
+ }
+
+ /*
+ * Pass condition: only document with id 6 is collapsed
+ */
+ public void testAdjacentCollapse_CollapseThresholdTwo() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map params = new HashMap();
+ params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"});
+ params.put(CollapseParams.COLLAPSE_THRESHOLD, new String[]{"2"});
+ responseBuilder.req.setParams(new ModifiableSolrParams(params));
+
+ collapseFilter = new AdjacentDocumentCollapser(responseBuilder);
+ // adjacent collapsing can only works with an order docset
+ String[] values = new String[]{"a", "b", "c", "z"};
+ int[] order = new int[]{3, 0, 0, 2, 1, 1, 1};
+ FieldCache.StringIndex index = new FieldCache.StringIndex(order, values);
+ int[] docIds = new int[]{0, 1, 2, 3, 4, 5, 6};
+ DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7);
+ collapseFilter.doCollapsing(uncollapsedDocset, index);
+
+ Map collapseCounts = collapseFilter.getCollapseCounts();
+ assertEquals(1, collapseCounts.size());
+ assertEquals(1, (int) collapseCounts.get(4));
+
+ DocSet docHeadsDocSet = collapseFilter.createDocumentCollapseResult().getCollapsedDocset();
+ assertEquals(6, docHeadsDocSet.size());
+
+ boolean zeroFound = false;
+ boolean oneFound = false;
+ boolean twoFound = false;
+ boolean threeFound = false;
+ boolean fourFound = false;
+ boolean fiveFound = false;
+ boolean sixFound = false;
+ for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) {
+ int docId = i.nextDoc();
+ if (docId == 0) {
+ zeroFound = true;
+ } else if (docId == 1) {
+ oneFound = true;
+ } else if (docId == 2) {
+ twoFound = true;
+ } else if (docId == 3) {
+ threeFound = true;
+ } else if (docId == 4) {
+ fourFound = true;
+ } else if (docId == 5) {
+ fiveFound = true;
+ } else if (docId == 6) {
+ sixFound = true;
+ }
+ }
+
+ assertTrue(zeroFound);
+ assertTrue(oneFound);
+ assertTrue(twoFound);
+ assertTrue(threeFound);
+ assertTrue(fourFound);
+ assertTrue(fiveFound);
+ assertFalse(sixFound);
+ }
+
+}
Index: src/java/org/apache/solr/search/DocumentCollapser.java
===================================================================
--- src/java/org/apache/solr/search/DocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/search/DocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * A document collapser is responsible for removing documents that have the same value on a predefined field,
+ * only the most relevant (based sorting and / or scoring) documents will not be removed. filterQueries, Sort sort) throws IOException;
+
+ /**
+ * Returns the statistics about the collapsed DocSet.
+ * The following information is returned:
+ *
+ * The fieldname used during collapsing
+ * A list of head document ids and how many documents were collapsed under it
+ * A list of field values and how many documents were collapsed under under it. The field values are from the
+ * collapse field and
+ * Debug information
+ *
+ *
+ * This method may only be invoked when the {@link this#collapse(Query, List, Sort)} has been invoked.
+ *
+ * @param searcher The solr index searcher
+ * @param docs The doclist containing the results to be displayed
+ * @return collapse counts for all documents in the specified docList.
+ * @throws IOException if index searcher related problems occur
+ */
+ NamedList getCollapseInfo(SolrIndexSearcher searcher, DocList docs) throws IOException;
+
+}
Index: src/test/org/apache/solr/search/FieldCollapsingIntegrationTest.java
===================================================================
--- src/test/org/apache/solr/search/FieldCollapsingIntegrationTest.java Sat Sep 12 12:08:36 CEST 2009
+++ src/test/org/apache/solr/search/FieldCollapsingIntegrationTest.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.CollapseParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.request.SolrQueryResponse;
+import org.apache.solr.update.AddUpdateCommand;
+import org.apache.solr.util.AbstractSolrTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Integration tests for field collapsing functionality.
+ */
+public class FieldCollapsingIntegrationTest extends AbstractSolrTestCase {
+
+ public String getSchemaFile() {
+ return "schema-fieldcollapse.xml";
+ }
+
+ public String getSolrConfigFile() {
+ return "solrconfig-fieldcollapse.xml";
+ }
+
+ public void testNormalFieldCollapse() throws Exception {
+ addToIndex("id", "1", "name", "author1", "title", "a tree");
+ addToIndex("id", "2", "name", "author1", "title", "a tree with a tree");
+ addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree");
+ addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree");
+ addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree");
+ assertU(commit());
+
+ SolrQueryRequest request = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put("q", new String[]{"tree"});
+ request.setParams(new ModifiableSolrParams(parameters));
+
+ SolrQueryResponse response = h.queryAndResponse(null, request);
+ DocSlice docSlice = (DocSlice) response.getValues().get("response");
+ assertEquals(2, docSlice.docs.length);
+
+ NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts");
+ assertEquals("name_plain", collapseCounts.get("field"));
+
+ NamedList fieldValueCounts = (NamedList) collapseCounts.get("count");
+ assertEquals(2, fieldValueCounts.size());
+ assertEquals(2, fieldValueCounts.get("author1"));
+ assertEquals(1, fieldValueCounts.get("author2"));
+
+ NamedList docIdCounts = (NamedList) collapseCounts.get("doc");
+ assertEquals(2, docIdCounts.size());
+ assertEquals(2, docIdCounts.get("1"));
+ assertEquals(1, docIdCounts.get("4"));
+ }
+
+ public void testNormalFieldCollapse_withFacetingBefore() throws Exception {
+ addToIndex("id", "1", "name", "author1", "title", "a tree");
+ addToIndex("id", "2", "name", "author1", "title", "a tree with a tree");
+ addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree");
+ addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree");
+ addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree");
+ assertU(commit());
+
+ SolrQueryRequest request = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()});
+ parameters.put("facet", new String[]{"true"});
+ parameters.put("facet.field", new String[]{"name"});
+ parameters.put("q", new String[]{"tree"});
+ parameters.put("fq", new String[]{"*:*"});
+ request.setParams(new ModifiableSolrParams(parameters));
+
+ SolrQueryResponse response = h.queryAndResponse(null, request);
+ DocSlice docSlice = (DocSlice) response.getValues().get("response");
+ assertEquals(2, docSlice.docs.length);
+
+ SimpleOrderedMap facetCounts = (SimpleOrderedMap) response.getValues().get("facet_counts");
+ SimpleOrderedMap facetFields = (SimpleOrderedMap) facetCounts.get("facet_fields");
+ NamedList counts = (NamedList) facetFields.get("name");
+ assertEquals(3, counts.get("author1"));
+ assertEquals(2, counts.get("author2"));
+ }
+
+ public void testNormalFieldCollapse_withFacetingAfter() throws Exception {
+ addToIndex("id", "1", "name", "author1", "title", "a tree");
+ addToIndex("id", "2", "name", "author1", "title", "a tree with a tree");
+ addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree");
+ addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree");
+ addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree");
+ assertU(commit());
+
+ SolrQueryRequest request = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.AFTER.toString()});
+ parameters.put("facet", new String[]{"true"});
+ parameters.put("facet.field", new String[]{"name"});
+ parameters.put("q", new String[]{"tree"});
+ request.setParams(new ModifiableSolrParams(parameters));
+
+ SolrQueryResponse response = h.queryAndResponse(null, request);
+ DocSlice docSlice = (DocSlice) response.getValues().get("response");
+ assertEquals(2, docSlice.docs.length);
+
+ SimpleOrderedMap facetCounts = (SimpleOrderedMap) response.getValues().get("facet_counts");
+ SimpleOrderedMap facetFields = (SimpleOrderedMap) facetCounts.get("facet_fields");
+ NamedList counts = (NamedList) facetFields.get("name");
+ assertEquals(1, counts.get("author1"));
+ assertEquals(1, counts.get("author2"));
+ }
+
+ public void testAdjacentFieldCollapse() throws Exception {
+ addToIndex("id", "1", "name", "author1", "title", "a tree");
+ addToIndex("id", "2", "name", "author1", "title", "a tree with a tree");
+ addToIndex("id", "3", "name", "author2", "title", "tree tree and yet another tree and a random word");
+ addToIndex("id", "4", "name", "author2", "title", "a lot of words and a tree");
+ addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree");
+ assertU(commit());
+
+ SolrQueryRequest request = lrf.makeRequest();
+ Map parameters = new HashMap();
+ parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{"adjacent"});
+ parameters.put(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS, new String[]{"true"});
+ parameters.put(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, new String[]{"id,title"});
+ parameters.put("q", new String[]{"tree"});
+ request.setParams(new ModifiableSolrParams(parameters));
+
+ SolrQueryResponse response = h.queryAndResponse(null, request);
+ DocSlice docSlice = (DocSlice) response.getValues().get("response");
+ assertEquals(3, docSlice.docs.length);
+
+ NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts");
+ assertEquals("name_plain", collapseCounts.get("field"));
+
+ NamedList fieldValueCounts = (NamedList) collapseCounts.get("count");
+ assertEquals(2, fieldValueCounts.size());
+ assertEquals(1, fieldValueCounts.get("author1"));
+ assertEquals(1, fieldValueCounts.get("author2"));
+
+ NamedList docIdCounts = (NamedList) collapseCounts.get("doc");
+ assertEquals(2, docIdCounts.size());
+ assertEquals(1, docIdCounts.get("1"));
+ assertEquals(1, docIdCounts.get("3"));
+
+ NamedList collapsedDocs = (NamedList) collapseCounts.get("collapsedDocs");
+ assertEquals(2, collapsedDocs.size());
+
+ assertEquals(1, collapsedDocs.get("author1").getNumFound());
+ assertEquals(2, collapsedDocs.get("author1").get(0).getFieldValue("id"));
+ assertEquals("a tree with a tree", collapsedDocs.get("author1").get(0).getFieldValue("title"));
+ assertNull(collapsedDocs.get("author1").get(0).getFieldValue("author"));
+
+ assertEquals(1, collapsedDocs.get("author2").getNumFound());
+ assertEquals(4, collapsedDocs.get("author2").get(0).getFieldValue("id"));
+ assertEquals("a lot of words and a tree", collapsedDocs.get("author2").get(0).getFieldValue("title"));
+ assertNull(collapsedDocs.get("author2").get(0).getFieldValue("author"));
+ }
+
+
+ // ================================================= Helpers =======================================================
+
+ protected void addToIndex(SolrInputDocument document) throws IOException {
+ SolrQueryRequest request = lrf.makeRequest();
+ SolrQueryResponse response = new SolrQueryResponse();
+ AddUpdateCommand command = new AddUpdateCommand();
+ command.solrDoc = document;
+ request.getCore().getUpdateProcessingChain(null).createProcessor(request, response).processAdd(command);
+ }
+
+ protected void addToIndex(String... fields) throws IOException {
+ SolrInputDocument document = createSolrInputDocument(fields);
+ addToIndex(document);
+ }
+
+ protected SolrInputDocument createSolrInputDocument(String... fields) {
+ if (fields.length % 2 != 0) {
+ throw new IllegalArgumentException("Supply both the name and values, field array length must be even");
+ }
+
+ SolrInputDocument document = new SolrInputDocument();
+ for (int i = 0; i < fields.length; i++) {
+ document.addField(fields[i], fields[++i]);
+ }
+
+ return document;
+ }
+}
Index: src/java/org/apache/solr/search/DocumentCollapseResult.java
===================================================================
--- src/java/org/apache/solr/search/DocumentCollapseResult.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/search/DocumentCollapseResult.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.search;
+
+/**
+ * Contains the collapsed and uncollapsed result.
+ */
+public class DocumentCollapseResult {
+
+ private final DocSet collapsedDocset;
+ private final DocSet unCollapsedDocset;
+
+ /**
+ * Constructs a DocumentCollapseResult.
+ *
+ * @param collapsedDocset The collapsed docset
+ * @param unCollapsedDocset The uncollapsed docset
+ */
+ public DocumentCollapseResult(DocSet collapsedDocset, DocSet unCollapsedDocset) {
+ this.collapsedDocset = collapsedDocset;
+ this.unCollapsedDocset = unCollapsedDocset;
+ }
+
+ /**
+ * Returns the collapsed docset.
+ *
+ * @return the collapsed docset
+ */
+ public DocSet getCollapsedDocset() {
+ return collapsedDocset;
+ }
+
+ /**
+ * Returns the uncollapsed docset.
+ *
+ * @return the uncollapsed docset
+ */
+ public DocSet getUnCollapsedDocset() {
+ return unCollapsedDocset;
+ }
+}
Index: src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java
===================================================================
--- src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java Sat Sep 12 12:08:36 CEST 2009
+++ src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,219 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.NamedList;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Represents the fieldcollapse part of the response.
+ * This class encapsulates the following information:
+ *
+ * What field was used during field collapsing.
+ * How many documents were collapsed under a certain field value
+ * How many documents were collapsed under a group head
+ * The collapsed documents per field value.
+ *
+ */
+public class FieldCollapseResponse {
+
+ private final String collapseField;
+ private final List fieldValueCollapseCounts;
+ private final List documentIdCollapseCounts;
+ private final Map collapsedDocuments;
+
+ /**
+ * Constructs a FieldCollapseResponse instance.
+ *
+ * @param collapseInfo The field collapse part of the response. From the namelist collapse_counts
+ */
+ public FieldCollapseResponse(NamedList collapseInfo) {
+ this.collapseField = (String) collapseInfo.get("field");
+ this.fieldValueCollapseCounts = parseFieldValueCollapseCounts((NamedList) collapseInfo.get("count"));
+ this.documentIdCollapseCounts = parseDocumentIdCollapseCounts((NamedList) collapseInfo.get("doc"));
+ this.collapsedDocuments = parseCollapsedDocuments((NamedList) collapseInfo.get("collapsedDocs"));
+ }
+
+ /**
+ * Returns the field name that was used during collapsing.
+ *
+ * @return the field name that was used during collapsing
+ */
+ public String getCollapseField() {
+ return collapseField;
+ }
+
+ /**
+ * Returns a list of {@link FieldValueCollapseCount}. That contains information about how many
+ * documents were collapsed under a certainer field value.
+ *
+ * @return a list of {@link FieldValueCollapseCount}
+ */
+ public List getFieldValueCollapseCounts() {
+ return fieldValueCollapseCounts;
+ }
+
+ /**
+ * Returns a list of {@link DocumentIdCollapseCount}, that contains information about how many
+ * documents were collapsed under a certain group head.
+ *
+ * @return a list of {@link DocumentIdCollapseCount}
+ */
+ public List getDocumentIdCollapseCounts() {
+ return documentIdCollapseCounts;
+ }
+
+ /**
+ * Returns all collapsed document as SolrDocumentList per field value in a map structure.
+ *
+ * Returns an empty map when no documents were collapsed or the inclusion of collapsed documents
+ * was not specified in the request
+ *
+ * @return all collapsed documents per field value
+ */
+ public Map getCollapsedDocuments() {
+ return collapsedDocuments;
+ }
+
+
+ // ================================================ Helper Methods =================================================
+
+ private List parseFieldValueCollapseCounts(NamedList info) {
+ List collapseCounts = new ArrayList();
+
+ for (int i = 0; i < info.size(); i++) {
+ String fieldName = info.getName(i);
+ int count = info.getVal(i);
+ collapseCounts.add(new FieldValueCollapseCount(fieldName, count));
+ }
+
+ return collapseCounts;
+ }
+
+ private List parseDocumentIdCollapseCounts(NamedList info) {
+ List collapseCounts = new ArrayList();
+
+ for (int i = 0; i < info.size(); i++) {
+ long documentId = Long.parseLong(info.getName(i));
+ int count = info.getVal(i);
+ collapseCounts.add(new DocumentIdCollapseCount(documentId, count));
+ }
+
+ return collapseCounts;
+ }
+
+ private Map parseCollapsedDocuments(NamedList info) {
+ Map collapsedDocuments = new HashMap();
+ if (info == null) {
+ return collapsedDocuments;
+ }
+
+ for (int i = 0; i < info.size(); i++) {
+ String fieldValue = info.getName(i);
+ SolrDocumentList documentList = info.getVal(i);
+ collapsedDocuments.put(fieldValue, documentList);
+ }
+
+ return collapsedDocuments;
+ }
+
+
+ // ================================================= Inner classes =================================================
+
+ /**
+ * Represents how many documents where collapsed under a certain field value.
+ */
+ public class FieldValueCollapseCount {
+
+ private final String fieldValue;
+ private final int count;
+
+ /**
+ * Constructs FieldValueCollapseCount.
+ *
+ * @param fieldValue The collapsed field value
+ * @param count The number of documents that collapsed under the specified field value.
+ */
+ public FieldValueCollapseCount(String fieldValue, int count) {
+ this.fieldValue = fieldValue;
+ this.count = count;
+ }
+
+ /**
+ * Returns the collapsed field value.
+ *
+ * @return the collapsed field value
+ */
+ public String getFieldValue() {
+ return fieldValue;
+ }
+
+ /**
+ * Returns the number of documents that collapsed under the specified field value.
+ *
+ * @return the number of documents that collapsed under the specified field value
+ */
+ public int getCount() {
+ return count;
+ }
+ }
+
+ /**
+ * Represents the head document id and how many documents were collapsed under it.
+ */
+ public class DocumentIdCollapseCount {
+
+ private final long documentId;
+ private final int count;
+
+ /**
+ * Constructs DocumentIdCollapseCount.
+ *
+ * @param documentId The head document identifier
+ * @param count The number of documents that were collapsed under the head document
+ */
+ public DocumentIdCollapseCount(long documentId, int count) {
+ this.documentId = documentId;
+ this.count = count;
+ }
+
+ /**
+ * Returns the head document id.
+ * This is the most relevant document of the group (documents with a common field value) and thus was not collapsed.
+ *
+ * @return head document id
+ */
+ public long getDocumentId() {
+ return documentId;
+ }
+
+ /**
+ * Returns the number of documents that were collapsed under the document head.
+ *
+ * @return the number of documents that were collapsed under the document head
+ */
+ public int getCount() {
+ return count;
+ }
+ }
+}
Index: src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java
===================================================================
--- src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java Sat Sep 12 12:08:36 CEST 2009
+++ src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.response;
+
+import junit.framework.TestCase;
+import org.apache.solr.client.solrj.ResponseParser;
+import org.apache.solr.client.solrj.impl.XMLResponseParser;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Tests for {@link org.apache.solr.client.solrj.response.FieldCollapseResponse}
+ */
+public class FieldCollapseResponseTest extends TestCase {
+
+ /*
+ * Pass condition: expected values in test file must appear in FieldCollapseResponse
+ */
+ public void testFieldCollapseResponse() throws Exception {
+ InputStream testDataStream = new FileInputStream(new File("fieldcollapse/testResponse.xml"));
+ ResponseParser responseParser = new XMLResponseParser();
+ NamedList response = responseParser.processResponse(new InputStreamReader(testDataStream, "UTF-8"));
+
+ QueryResponse queryResponse = new QueryResponse();
+ queryResponse.setResponse(response);
+ FieldCollapseResponse result = queryResponse.getFieldCollapseResponse();
+
+ assertNotNull(result);
+ assertEquals("venue", result.getCollapseField());
+
+ assertEquals(1, result.getFieldValueCollapseCounts().size());
+ assertEquals("melkweg", result.getFieldValueCollapseCounts().get(0).getFieldValue());
+ assertEquals(1, result.getFieldValueCollapseCounts().get(0).getCount());
+
+ assertEquals(1, result.getDocumentIdCollapseCounts().size());
+ assertEquals(233238L, result.getDocumentIdCollapseCounts().get(0).getDocumentId());
+ assertEquals(1, result.getDocumentIdCollapseCounts().get(0).getCount());
+
+ assertEquals(1, result.getCollapsedDocuments().size());
+ assertEquals(1, result.getCollapsedDocuments().get("melkweg").getNumFound());
+ assertEquals("Amsterdam", result.getCollapsedDocuments().get("melkweg").get(0).getFieldValue("city"));
+ }
+}
Index: src/java/org/apache/solr/util/DocSetScoreCollector.java
===================================================================
--- src/java/org/apache/solr/util/DocSetScoreCollector.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/util/DocSetScoreCollector.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.util;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.Collector;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.util.OpenBitSet;
+import org.apache.solr.search.BitDocSet;
+import org.apache.solr.search.DocSet;
+
+import java.io.IOException;
+
+/**
+ * Collects the documents with the score.
+ */
+public class DocSetScoreCollector extends Collector {
+ private final float[] scores;
+ private final OpenBitSet bits;
+
+ private Scorer scorer = null;
+ private int docBase = 0;
+
+ /**
+ * Constructs a DocSetHitCollector with the specified parameter.
+ *
+ * @param maxDoc The maximum size of all the documents in the index
+ */
+ public DocSetScoreCollector(int maxDoc) {
+ scores = new float[maxDoc];
+ bits = new OpenBitSet(maxDoc);
+ }
+
+ /**
+ * Collects documents as specified in {@link super#collect(int)}.
+ * Also stores the score associated with this document.
+ *
+ * @param doc The document id to collect
+ * @throws IOException
+ */
+ public void collect(int doc) throws IOException {
+ doc = doc + docBase;
+ bits.fastSet(doc);
+ scores[doc] = scorer.score();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public boolean acceptsDocsOutOfOrder() {
+ return false;
+ }
+
+ // ================================================= Setter/Getter ===================================================
+
+ /**
+ * {@inheritDoc}
+ */
+ public void setNextReader(IndexReader reader, int docBase) throws IOException {
+ this.docBase = docBase;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public void setScorer(Scorer scorer) throws IOException {
+ this.scorer = scorer;
+ }
+
+ /**
+ * Returns a docset of the collected documents.
+ *
+ * @return a docset of the collected documents
+ */
+ public DocSet getDocSet() {
+ return new BitDocSet(bits);
+ }
+
+ /**
+ * Returns the scores for the collected documents.
+ * The index of the score represents the lucene document identifier.
+ *
+ * @return the scores for the collected documents
+ */
+ public float[] getScores() {
+ return scores;
+ }
+
+}
Index: src/java/org/apache/solr/search/NonAdjacentDocumentCollapser.java
===================================================================
--- src/java/org/apache/solr/search/NonAdjacentDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
+++ src/java/org/apache/solr/search/NonAdjacentDocumentCollapser.java Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,495 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.search;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.*;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.solr.handler.component.ResponseBuilder;
+import org.apache.solr.util.DocSetScoreCollector;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Normal collapsing behaviour. Collapses all documents that have the same field value on the predefined field
+ * and that have occured equal or more then collapse threshold.
+ */
+public class NonAdjacentDocumentCollapser extends AbstractDocumentCollapser {
+
+ DocumentComparator documentComparator;
+
+ /**
+ * See {@link super#AbstractDocumentCollapser(org.apache.solr.handler.component.ResponseBuilder)}
+ */
+ public NonAdjacentDocumentCollapser(ResponseBuilder rb) throws IOException {
+ super(rb);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException {
+ if (containsSortOnScore(sort)) {
+ DocSetScoreCollector docSetCollector = new DocSetScoreCollector(searcher.maxDoc());
+ Filter filter = null;
+ if (filterQueries != null && filterQueries.size() > 0) {
+ filter = new QueryWrapperFilter(createBooleanQuery(filterQueries));
+ }
+
+ long startTime = System.currentTimeMillis();
+ searcher.search(mainQuery, filter, docSetCollector);
+ timeCreateUncollapedDocset = System.currentTimeMillis() - startTime;
+ uncollapsedDocSet = docSetCollector.getDocSet();
+ if (sort == null) {
+ sort = new Sort(new SortField("score", SortField.SCORE, true));
+ }
+
+ documentComparator = new DocumentComparator(sort, searcher.getIndexReader().numDocs(), searcher.getIndexReader(), docSetCollector.getScores());
+ } else {
+ long startTime = System.currentTimeMillis();
+ uncollapsedDocSet = searcher.getDocSet(merge(mainQuery, filterQueries));
+ timeCreateUncollapedDocset = System.currentTimeMillis() - startTime;
+
+ documentComparator = new DocumentComparator(sort, searcher.getIndexReader().numDocs(), searcher.getIndexReader());
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ protected void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values) {
+ long startTime = System.currentTimeMillis();
+ int docCount = 0;
+ // Keep how many documents we have processed the track of how many docs
+ // with the same collapse value we have processed so far.
+ Map collapsedDocs = new HashMap();
+
+ for (DocIterator i = uncollapsedDocset.iterator(); i.hasNext();) {
+ int currentId = i.nextDoc();
+ String currentValue = values.lookup[values.order[currentId]];
+
+ // Get the last doc. and the total amount of docs. we have seen so
+ // far for this collapsing value
+ CollapsedDocumentGroup collapseDoc = collapsedDocs.get(currentValue);
+ if (collapseDoc == null) {
+ // new collapsing value => create a new record for it
+ collapseDoc = new CollapsedDocumentGroup(0, 0, documentComparator, collapseTreshold);
+ collapsedDocs.put(currentValue, collapseDoc);
+ }
+ collapseDoc.priorityQueue.insertWithOverflow(currentId);
+
+ // check if we have reached the collapse threshold, if so start counting collapsed documents
+ if (++collapseDoc.totalCount > collapseTreshold) {
+ collapseDoc.collapsedDocuments++;
+ addCollapsedDoc(currentId, currentValue);
+ }
+
+ // Stop after collapseMaxDocs documents
+ if (++docCount >= collapseMaxDocs) {
+ break;
+ }
+ }
+
+ // adding the head documents to the internal document buffer and
+ // adding the collapsed counts per document head to the map
+ for (CollapsedDocumentGroup collapseDoc : collapsedDocs.values()) {
+ if (collapseDoc.collapsedDocuments > 0) {
+ collapseCounts.put((Integer) collapseDoc.priorityQueue.top(), collapseDoc.collapsedDocuments);
+ }
+ Integer doc;
+ while ((doc = (Integer) collapseDoc.priorityQueue.pop()) != null) {
+ addDoc(doc);
+ }
+ }
+
+ timeCollapsing = System.currentTimeMillis() - startTime;
+ }
+
+
+ // ================================================= Helpers =======================================================
+
+ /**
+ * Returns true if the sort contains a sortfield that sorts on score, otherwise false.
+ *
+ * @param sort The sort
+ * @return true if the sort contains a sortfield that sorts on score, otherwise false
+ */
+ protected boolean containsSortOnScore(Sort sort) {
+ if (sort == null) {
+ return true; // means default sorting, which is sorting on score desc
+ }
+
+ for (SortField field : sort.getSort()) {
+ if (field.getType() == SortField.SCORE) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns a boolean query that wraps the specified queries in a BooleanQuery with a must claus.
+ *
+ * @param queries The filter queries
+ * @return a boolean query that wraps the specified queries in a BooleanQuery with a must clause.
+ */
+ protected BooleanQuery createBooleanQuery(List queries) {
+ if (queries == null || queries.size() == 0) {
+ return null;
+ }
+
+ BooleanQuery booleanQuery = new BooleanQuery();
+ for (Query filterQuery : queries) {
+ booleanQuery.add(filterQuery, BooleanClause.Occur.MUST);
+ }
+ return booleanQuery;
+ }
+
+ protected List merge(Query mainQuery, List filterQueries) {
+ if (filterQueries == null) {
+ List queries = new ArrayList();
+ queries.add(mainQuery);
+ return queries;
+ } else {
+ filterQueries.add(mainQuery);
+ return filterQueries;
+ }
+ }
+
+
+ // ============================================ Inner Classes ======================================================
+
+ /**
+ * A PriorityQueue that maintaince order with a DocumentComparator.
+ */
+ public static class DocumentPriorityQueue extends PriorityQueue {
+
+ private final DocumentComparator comparator;
+
+ /**
+ * Constructs a DocumentPriorityQueue
+ *
+ * @param comparator The DocumentComparator used for maintaining order in the queue
+ * @param max The maximum number of document identifiers in the queue (is equal to collapse thresold paramter)
+ */
+ public DocumentPriorityQueue(DocumentComparator comparator, int max) {
+ this.comparator = comparator;
+ initialize(max);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ protected boolean lessThan(Object a, Object b) {
+ return comparator.compare((Integer) a, (Integer) b) < 0;
+ }
+ }
+
+ /**
+ * Compares two documents with each other.
+ */
+ public static class DocumentComparator {
+
+ private final FieldComparator[] fieldComparators;
+ private final boolean[] descending;
+ private Scorer scorer;
+
+ /**
+ * Constructs a DocumentComparator by initializing the
+ * {@link org.apache.lucene.search.FieldComparator}s and determining the sort orders.
+ *
+ * @param sort The sort used for the creation of the FieldComparators.
+ * @param numberOfHits The number of results in the pre-field-collapsed resultset
+ * @param reader The index reader, used for reading field values (in the FieldComparators)
+ */
+ public DocumentComparator(Sort sort, int numberOfHits, IndexReader reader) {
+ fieldComparators = new FieldComparator[sort.getSort().length];
+ descending = new boolean[sort.getSort().length];
+ initializeFieldComparators(sort, numberOfHits, reader, null);
+ }
+
+ /**
+ * Constructs a DocumentComparator by initializing the
+ * {@link org.apache.lucene.search.FieldComparator}s and determining the sort orders.
+ *
+ * @param sort The sort used for the creation of the FieldComparators.
+ * @param numberOfHits The number of results in the pre-field-collapsed resultset
+ * @param reader The index reader, used for reading field values (in the FieldComparators)
+ * @param scores The scores used for comparing the documents
+ */
+ public DocumentComparator(Sort sort, int numberOfHits, IndexReader reader, float[] scores) {
+ fieldComparators = new FieldComparator[sort.getSort().length];
+ descending = new boolean[sort.getSort().length];
+ initializeFieldComparators(sort, numberOfHits, reader, scores);
+ }
+
+ /**
+ * Compares doc1 and doc2 with each other.
+ * Compares the two documents with the initialized FieldComparators,
+ * if all FieldComparators compares the documents as equal then the document with lowest lucene
+ * identifier will be classified as most relevant.
+ *
+ * @param doc1 The lucene identifier of the first document
+ * @param doc2 The lucene identifier of the second document
+ * @return -1 if doc1 is less relevant or equal relevant but has a higher lucene id then doc2,
+ * 0 if both documents are identical
+ * 1 if doc1 is more relevant or equal relevant but has a lower lucene id then doc2
+ */
+ public int compare(int doc1, int doc2) {
+ int result;
+ for (int i = 0; i < fieldComparators.length; i++) {
+ FieldComparator fieldComparator = fieldComparators[i];
+ try {
+ scorer.advance(doc2);
+ fieldComparator.copy(doc2, doc2);
+ scorer.advance(doc1);
+ fieldComparator.copy(doc1, doc1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ result = fieldComparator.compare(doc1, doc2);
+ result = descending[i] ? result : -result;
+
+ if (result != 0) {
+ return result;
+ }
+ }
+
+ // field comparators identified the field(s) values as equal
+ // Document with lowest identifier has higher precendence
+ if (doc1 < doc2) {
+ return 1;
+ } else if (doc1 > doc2) {
+ return -1;
+ }
+
+ // can only happen if comparing two the exact same document (with same lucene id)
+ return 0;
+ }
+
+ private void initializeFieldComparators(Sort sort, int numberOfHits, IndexReader indexReader, float[] scores) {
+ try {
+ scorer = new PredefinedScorer(scores == null ? new float[0] : scores);
+ if (sort.getSort().length == 1 && scores != null) {
+ fieldComparators[0] = new FloatValueFieldComparator(scores);
+ descending[0] = sort.getSort()[0].getReverse();
+ return;
+ }
+
+ for (int i = 0; i < sort.getSort().length; i++) {
+ SortField sortField = sort.getSort()[i];
+ fieldComparators[i] = sortField.getComparator(numberOfHits, i);
+ descending[i] = sortField.getReverse();
+ fieldComparators[i].setNextReader(indexReader, 0);
+ fieldComparators[i].setScorer(scorer);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ /**
+ * Represents a collapse group, that collects statistics for a certain fieldvalue group.
+ *
+ * Keeps track of the following statistics during the collapsing on fieldvalue:
+ *
+ * how many documents were collapsed
+ * what the most relevant head documents are in this group (these will not get collapsed)
+ * how many have been procesed
+ *
+ */
+ private static class CollapsedDocumentGroup {
+
+ int collapsedDocuments;
+ int totalCount;
+ final DocumentPriorityQueue priorityQueue;
+
+ /**
+ * Constructs a CollapsedDocumentGroup. Keeps track of the of the most relevant documents
+ * in this group. These documents will stay in the resultset and do not get collaped, no more then the
+ * specified collpasThreshold will be kept inside this CollapsedDocumentGroup.
+ *
+ * @param totalCount the total amount documents processed in the collapsing process
+ * @param collapsedDocuments the amount of documents collapsed under this group
+ * @param comparator The document comparater used inside priority queue
+ * @param collapsThreshold The threshold to start collapsing from
+ */
+ private CollapsedDocumentGroup(int totalCount, int collapsedDocuments, DocumentComparator comparator, int collapsThreshold) {
+ this.totalCount = totalCount;
+ this.collapsedDocuments = collapsedDocuments;
+ this.priorityQueue = new DocumentPriorityQueue(comparator, collapsThreshold);
+ }
+ }
+
+ /**
+ * A scorer that returns scores from a predefined array of scores.
+ */
+ private static final class PredefinedScorer extends Scorer {
+
+ private final float[] scores;
+ private int index;
+
+ private PredefinedScorer(float[] scores) {
+ super(null);
+ this.scores = scores;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public float score() throws IOException {
+ return scores[index];
+ }
+
+ /**
+ * Unsupported because this scorer contains predefined calculated scores, that cannot be explained.
+ *
+ * @param doc The document identifier
+ * @return unsupported operation exception
+ * @throws IOException
+ */
+ public Explanation explain(int doc) throws IOException {
+ throw new UnsupportedOperationException("Unsupported method");
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public int docID() {
+ return index;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public int nextDoc() throws IOException {
+ return index < scores.length ? ++index : NO_MORE_DOCS;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ public int advance(int target) throws IOException {
+ if (target < scores.length) {
+ index = target;
+ return index;
+ }
+ return NO_MORE_DOCS;
+ }
+
+ }
+
+ /**
+ * Field comparator for floats. This implementation is a little bit faster (around 30 ms) then {@link FloatComparator.FloatComparator}
+ * because it does not copy floats in {@link #copy(int, int)}. This is not necessary in fieldcollapsing case, because
+ * the arguments passed to the {@link #compare(int, int)} method are document ids and the values array is sorted on
+ * document order.
+ */
+ private static class FloatValueFieldComparator extends FieldComparator {
+
+ private final float[] values;
+
+ private FloatValueFieldComparator(float[] values) {
+ this.values = values;
+ }
+
+ /**
+ * Compares value for doc1 with value for doc2.
+ *
+ * @param doc1 first document to compare
+ * @param doc2 second document to compare
+ * @return any N < 0 if doc2's value is sorted after
+ * doc1, any N > 0 if the doc2's value is sorted before
+ * doc1 and 0 if they are equal
+ */
+ public int compare(int doc1, int doc2) {
+ final float value1 = values[doc1];
+ final float value2 = values[doc2];
+
+ if (value1 > value2) {
+ return 1;
+ } else if (value1 < value2) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Unsupported, is not necessary for field collapsing
+ *
+ * @param doc The document identifier
+ * @throws UnsupportedOperationException
+ */
+ public void setBottom(int doc) {
+ throw new UnsupportedOperationException("Method not implemented");
+ }
+
+ /**
+ * Unsupported, is not necessary for field collapsing
+ *
+ * @param doc The document identifier
+ * @return nothing
+ * @throws UnsupportedOperationException
+ */
+ public int compareBottom(int doc) throws IOException {
+ throw new UnsupportedOperationException("Method not implemented");
+ }
+
+ /**
+ * {@inheritDoc}
+ * params = new HashMap();
+ params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"});
+ responseBuilder.req.setParams(new ModifiableSolrParams(params));
+
+ float[] scores = new float[]{1.0f, 2.0f, 0.1f, 1.5f, 2.5f, 0.1f, 0.1f};
+ Sort sort = new Sort(new SortField("", SortField.SCORE, true));
+ final NonAdjacentDocumentCollapser.DocumentComparator comparator =
+ new NonAdjacentDocumentCollapser.DocumentComparator(sort, scores.length, null, scores);
+ nonAdjacentCollapser = new NonAdjacentDocumentCollapser(responseBuilder) {
+
+ @Override
+ protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException {
+ this.documentComparator = comparator;
+ }
+ };
+
+ String[] values = new String[]{"a", "b", "c"};
+ int[] order = new int[]{0, 1, 0, 2, 1, 0, 1};
+ FieldCache.StringIndex index = new FieldCache.StringIndex(order, values);
+ int[] docIds = new int[]{1, 2, 0, 3, 4, 5, 6};
+ DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7);
+ nonAdjacentCollapser.doQuery(null, null, null);
+ nonAdjacentCollapser.doCollapsing(uncollapsedDocset, index);
+
+ Map collapseCounts = nonAdjacentCollapser.getCollapseCounts();
+ assertEquals(2, collapseCounts.size());
+ assertEquals(2, (int) collapseCounts.get(0));
+ assertEquals(2, (int) collapseCounts.get(4));
+
+ DocSet docHeadsDocSet = nonAdjacentCollapser.createDocumentCollapseResult().getCollapsedDocset();
+ assertEquals(3, docHeadsDocSet.size());
+
+ boolean zeroFound = false;
+ boolean oneFound = false;
+ boolean twoFound = false;
+ boolean threeFound = false;
+ boolean fourFound = false;
+ boolean fiveFound = false;
+ boolean sixFound = false;
+ for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) {
+ int docId = i.nextDoc();
+ if (docId == 0) {
+ zeroFound = true;
+ } else if (docId == 1) {
+ oneFound = true;
+ } else if (docId == 2) {
+ twoFound = true;
+ } else if (docId == 3) {
+ threeFound = true;
+ } else if (docId == 4) {
+ fourFound = true;
+ } else if (docId == 5) {
+ fiveFound = true;
+ } else if (docId == 6) {
+ sixFound = true;
+ }
+ }
+
+ assertTrue(zeroFound);
+ assertFalse(oneFound);
+ assertFalse(twoFound);
+ assertTrue(threeFound);
+ assertTrue(fourFound);
+ assertFalse(fiveFound);
+ assertFalse(sixFound);
+ }
+
+ public void testNormalCollapse_collapseThresholdThree() throws Exception {
+ ResponseBuilder responseBuilder = new ResponseBuilder();
+ responseBuilder.req = lrf.makeRequest();
+ Map params = new HashMap();
+ params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"});
+ params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"});
+ params.put(CollapseParams.COLLAPSE_THRESHOLD, new String[]{"3"});
+ responseBuilder.req.setParams(new ModifiableSolrParams(params));
+
+ float[] scores = new float[]{1.0f, 2.0f, 0.1f, 1.5f, 2.5f, 0.1f, 0.1f};
+ Sort sort = new Sort(new SortField("", SortField.SCORE, true));
+ final NonAdjacentDocumentCollapser.DocumentComparator comparator =
+ new NonAdjacentDocumentCollapser.DocumentComparator(sort, scores.length, null, scores);
+ nonAdjacentCollapser = new NonAdjacentDocumentCollapser(responseBuilder) {
+
+ @Override
+ protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException {
+ this.documentComparator = comparator;
+ }
+ };
+ String[] values = new String[]{"a", "b", "c"};
+ int[] order = new int[]{0, 0, 0, 2, 1, 0, 1};
+ FieldCache.StringIndex index = new FieldCache.StringIndex(order, values);
+ int[] docIds = new int[]{1, 2, 0, 3, 4, 5, 6};
+ DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7);
+ nonAdjacentCollapser.doQuery(null, null, null);
+ nonAdjacentCollapser.doCollapsing(uncollapsedDocset, index);
+
+ Map collapseCounts = nonAdjacentCollapser.getCollapseCounts();
+ assertEquals(1, collapseCounts.size());
+ assertEquals(1, (int) collapseCounts.get(2));
+
+ DocSet docHeadsDocSet = nonAdjacentCollapser.createDocumentCollapseResult().getCollapsedDocset();
+ assertEquals(6, docHeadsDocSet.size());
+
+ boolean zeroFound = false;
+ boolean oneFound = false;
+ boolean twoFound = false;
+ boolean threeFound = false;
+ boolean fourFound = false;
+ boolean fiveFound = false;
+ boolean sixFound = false;
+ for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) {
+ int docId = i.nextDoc();
+ if (docId == 0) {
+ zeroFound = true;
+ } else if (docId == 1) {
+ oneFound = true;
+ } else if (docId == 2) {
+ twoFound = true;
+ } else if (docId == 3) {
+ threeFound = true;
+ } else if (docId == 4) {
+ fourFound = true;
+ } else if (docId == 5) {
+ fiveFound = true;
+ } else if (docId == 6) {
+ sixFound = true;
+ }
+ }
+
+ assertTrue(zeroFound);
+ assertTrue(oneFound);
+ assertTrue(twoFound);
+ assertTrue(threeFound);
+ assertTrue(fourFound);
+ assertFalse(fiveFound);
+ assertTrue(sixFound);
+ }
+
+}
Index: src/test/test-files/fieldcollapse/testResponse.xml
===================================================================
--- src/test/test-files/fieldcollapse/testResponse.xml Sat Sep 12 12:08:36 CEST 2009
+++ src/test/test-files/fieldcollapse/testResponse.xml Sat Sep 12 12:08:36 CEST 2009
@@ -0,0 +1,33 @@
+
+
+
+ 0
+ 2
+
+ venue
+ title:test
+ title
+ collapse
+
+
+
+ venue
+
+ 1
+
+
+ 1
+
+
+
+
+ 213133
+ Amsterdam
+
+
+
+
+
+
+
+