Index: src/java/org/apache/solr/search/fieldcollapse/CollapseGroup.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/CollapseGroup.java Sun Oct 25 20:03:59 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/CollapseGroup.java Sun Oct 25 20:03:59 CET 2009 @@ -0,0 +1,37 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +/** + * A CollapseGroup represents a logical group where documents are collaped into. + * Per {@link DocumentCollapser} the definition of a collapse group may differ. + *

+ * The uniqueness of the group is specified by its hashcode and equals method. All {@link DocumentCollapser} implementations + * must have a proper implementation of these methods. + */ +public interface CollapseGroup { + + /** + * Returns the value for presentation purposes in the response. + * This may not be an unique value. + * + * @return the value for presentation purposes in the response + */ + String getKey(); + +} Index: src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml =================================================================== --- src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml Sun Oct 25 21:08:27 CET 2009 +++ src/test/test-files/solr/conf/solrconfig-fieldcollapse.xml Sun Oct 25 21:08:27 CET 2009 @@ -0,0 +1,488 @@ + + + + + + + + + + + + ${solr.data.dir:./solr/data} + + + + + false + 10 + + + + 32 + 2147483647 + 10000 + 1000 + 10000 + + + false + + + org.apache.lucene.index.LogByteSizeMergePolicy + + + org.apache.lucene.index.ConcurrentMergeScheduler + + 1000 + 10000 + + single + + + + + false + 10 + + 10 + 2147483647 + 10000 + org.apache.lucene.index.LogDocMergePolicy + + true + + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + true + + + + + + + 10 + + + + + + + + + + + + + + + + + + + + + + + + + + groupDocumentsCounts + groupFieldValue + groupDocumentsFields + groupAggregatedData + + + + + + + + + + + + + org.apache.solr.search.fieldcollapse.collector.aggregate.SumFunction + org.apache.solr.search.fieldcollapse.collector.aggregate.AverageFunction + org.apache.solr.search.fieldcollapse.collector.aggregate.MinFunction + org.apache.solr.search.fieldcollapse.collector.aggregate.MaxFunction + + + + + + true + + collapse + facet + mlt + highlight + stats + debug + + + + + + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + 1000 + 1.4142135 + 12 + foo + + + sqrt 2 + log 10 + + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + false + + + + lowerfilt + + + default + lowerfilt + spellchecker1 + true + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + + + + + termsComp + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + + + + + + + + tvComponent + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + max-age=30, public + + + + + solr + solrconfig.xml scheam.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + false + true + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + Index: src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollectorFactory.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollectorFactory.java Tue Oct 13 11:05:15 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollectorFactory.java Tue Oct 13 11:05:15 CEST 2009 @@ -0,0 +1,38 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.solr.request.SolrQueryRequest; + +/** + * A concrete CollapseCollectorFactory implementation is responsible for creating {@link CollapseCollector} + * instances based on the {@link SolrQueryRequest}. + */ +public interface CollapseCollectorFactory { + + /** + * Creates an instance of a CollapseCollector specified by the concrete subclass. + * The concrete subclass decides based on the specified request if an new instance has to be created and + * can return null for that matter. + * + * @param request The specified request + * @return an instance of a CollapseCollector or null + */ + CollapseCollector createCollapseCollector(SolrQueryRequest request); + +} Index: src/test/test-files/fieldcollapse/testResponse.xml =================================================================== --- src/test/test-files/fieldcollapse/testResponse.xml Tue Oct 20 16:07:30 CEST 2009 +++ src/test/test-files/fieldcollapse/testResponse.xml Tue Oct 20 16:07:30 CEST 2009 @@ -0,0 +1,33 @@ + + + + 0 + 2 + + venue + title:test + title + collapse + + + + venue + + 1 + + + 1 + + + + + 213133 + Amsterdam + + + + + + + + \ No newline at end of file Index: src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MaxFunction.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MaxFunction.java Sun Oct 18 23:40:34 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MaxFunction.java Sun Oct 18 23:40:34 CEST 2009 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector.aggregate; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; + +/** + * Computes the higest value per collapse group. + */ +public class MaxFunction implements AggregateFunction { + + private final Map higestNumbers = new HashMap(); + + public void collect(CollapseGroup collapseGroup, String number) { + BigDecimal higestNumber = higestNumbers.get(collapseGroup); + BigDecimal newNumber = new BigDecimal(number); + if (higestNumber == null) { + higestNumbers.put(collapseGroup, newNumber); + } else { + higestNumbers.put(collapseGroup, higestNumber.max(newNumber)); + } + } + + public String calculate(CollapseGroup collapseGroup) { + BigDecimal higestNumber = higestNumbers.get(collapseGroup); + return higestNumber != null ? higestNumber.toString() : null; + } + + public String getName() { + return "max"; + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MinFunction.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MinFunction.java Sun Oct 18 23:40:34 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/MinFunction.java Sun Oct 18 23:40:34 CEST 2009 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector.aggregate; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; + +/** + * Computes the lowest value per collapse group. + */ +public class MinFunction implements AggregateFunction { + + private final Map lowestNumbers = new HashMap(); + + public void collect(CollapseGroup collapseGroup, String number) { + BigDecimal lowestNumber = lowestNumbers.get(collapseGroup); + BigDecimal newNumber = new BigDecimal(number); + if (lowestNumber == null) { + lowestNumbers.put(collapseGroup, newNumber); + } else { + lowestNumbers.put(collapseGroup, lowestNumber.min(newNumber)); + } + } + + public String calculate(CollapseGroup collapseGroup) { + BigDecimal lowestNumber = lowestNumbers.get(collapseGroup); + return lowestNumber != null ? lowestNumber.toString() : null; + } + + public String getName() { + return "min"; + } + +} Index: src/java/org/apache/solr/core/SolrConfig.java =================================================================== --- src/java/org/apache/solr/core/SolrConfig.java (revision 824364) +++ src/java/org/apache/solr/core/SolrConfig.java Sun Oct 25 15:52:03 CET 2009 @@ -17,43 +17,40 @@ package org.apache.solr.core; +import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.search.BooleanQuery; import org.apache.solr.common.util.DOMUtil; -import org.apache.solr.common.util.RegexFileFilter; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.RegexFileFilter; import org.apache.solr.handler.PingRequestHandler; import org.apache.solr.handler.component.SearchComponent; +import org.apache.solr.highlight.SolrHighlighter; import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.QueryResponseWriter; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.request.QueryResponseWriter; - import org.apache.solr.search.CacheConfig; import org.apache.solr.search.FastLRUCache; import org.apache.solr.search.QParserPlugin; import org.apache.solr.search.ValueSourceParser; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; +import org.apache.solr.spelling.QueryConverter; import org.apache.solr.update.SolrIndexConfig; import org.apache.solr.update.processor.UpdateRequestProcessorChain; -import org.apache.solr.spelling.QueryConverter; -import org.apache.solr.highlight.SolrHighlighter; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.index.IndexDeletionPolicy; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import org.w3c.dom.Node; import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.xpath.XPathConstants; - -import java.util.*; -import java.util.regex.Pattern; -import java.util.regex.Matcher; import java.io.FileFilter; import java.io.IOException; import java.io.InputStream; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** @@ -193,6 +190,7 @@ loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true); loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false); loadPluginInfo(SolrHighlighter.class,"highlighting",false, false); + loadPluginInfo(CollapseCollectorFactory.class, "//fieldCollapsing/collapseCollectorFactory", true, true); updateHandlerInfo = loadUpdatehandlerInfo(); Config.log.info("Loaded SolrConfig: " + name); Index: src/test/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapserTest.java =================================================================== --- src/test/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapserTest.java Sun Oct 25 19:32:13 CET 2009 +++ src/test/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapserTest.java Sun Oct 25 19:32:13 CET 2009 @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.HashDocSet; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; +import org.apache.solr.search.fieldcollapse.collector.DocumentGroupCountCollapseCollectorFactory; +import org.apache.solr.search.fieldcollapse.util.Counter; +import org.apache.solr.util.AbstractSolrTestCase; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Unit tests for {@link NonAdjacentDocumentCollapser}. + */ +public class NonAdjacentDocumentCollapserTest extends AbstractSolrTestCase { + + private NonAdjacentDocumentCollapser nonAdjacentCollapser; + + public String getSchemaFile() { + return "schema-fieldcollapse.xml"; + } + + public String getSolrConfigFile() { + return "solrconfig-fieldcollapse.xml"; + } + + public void testNormalCollapse_collapseThresholdOne() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map params = new HashMap(); + params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"}); + responseBuilder.req.setParams(new ModifiableSolrParams(params)); + + float[] scores = new float[]{1.0f, 2.0f, 0.1f, 1.5f, 2.5f, 0.1f, 0.1f}; + Sort sort = new Sort(new SortField("", SortField.SCORE, true)); + final NonAdjacentDocumentCollapser.DocumentComparator comparator = + new NonAdjacentDocumentCollapser.DocumentComparator(sort, scores.length, null, scores); + CollapseCollectorFactory factory = new DocumentGroupCountCollapseCollectorFactory(); + nonAdjacentCollapser = new NonAdjacentDocumentCollapser(responseBuilder, Arrays.asList(factory)) { + + @Override + protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException { + this.documentComparator = comparator; + } + }; + + String[] values = new String[]{"a", "b", "c"}; + int[] order = new int[]{0, 1, 0, 2, 1, 0, 1}; + FieldCache.StringIndex index = new FieldCache.StringIndex(order, values); + int[] docIds = new int[]{1, 2, 0, 3, 4, 5, 6}; + DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7); + nonAdjacentCollapser.doQuery(null, null, null); + nonAdjacentCollapser.doCollapsing(uncollapsedDocset, index); + + Map collapseCounts = (Map) nonAdjacentCollapser.collapseContext.get("documentHeadCount"); + assertEquals(2, collapseCounts.size()); + assertEquals(2, collapseCounts.get(0).getCount()); + assertEquals(2, collapseCounts.get(4).getCount()); + + DocSet docHeadsDocSet = nonAdjacentCollapser.createDocumentCollapseResult().getCollapsedDocset(); + assertEquals(3, docHeadsDocSet.size()); + + boolean zeroFound = false; + boolean oneFound = false; + boolean twoFound = false; + boolean threeFound = false; + boolean fourFound = false; + boolean fiveFound = false; + boolean sixFound = false; + for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) { + int docId = i.nextDoc(); + if (docId == 0) { + zeroFound = true; + } else if (docId == 1) { + oneFound = true; + } else if (docId == 2) { + twoFound = true; + } else if (docId == 3) { + threeFound = true; + } else if (docId == 4) { + fourFound = true; + } else if (docId == 5) { + fiveFound = true; + } else if (docId == 6) { + sixFound = true; + } + } + + assertTrue(zeroFound); + assertFalse(oneFound); + assertFalse(twoFound); + assertTrue(threeFound); + assertTrue(fourFound); + assertFalse(fiveFound); + assertFalse(sixFound); + } + + public void testNormalCollapse_collapseThresholdThree() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map params = new HashMap(); + params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"}); + params.put(CollapseParams.COLLAPSE_THRESHOLD, new String[]{"3"}); + responseBuilder.req.setParams(new ModifiableSolrParams(params)); + + float[] scores = new float[]{1.0f, 2.0f, 0.1f, 1.5f, 2.5f, 0.1f, 0.1f}; + Sort sort = new Sort(new SortField("", SortField.SCORE, true)); + final NonAdjacentDocumentCollapser.DocumentComparator comparator = + new NonAdjacentDocumentCollapser.DocumentComparator(sort, scores.length, null, scores); + CollapseCollectorFactory factory = new DocumentGroupCountCollapseCollectorFactory(); + nonAdjacentCollapser = new NonAdjacentDocumentCollapser(responseBuilder, Arrays.asList(factory)) { + + @Override + protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException { + this.documentComparator = comparator; + } + }; + String[] values = new String[]{"a", "b", "c"}; + int[] order = new int[]{0, 0, 0, 2, 1, 0, 1}; + FieldCache.StringIndex index = new FieldCache.StringIndex(order, values); + int[] docIds = new int[]{1, 2, 0, 3, 4, 5, 6}; + DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7); + nonAdjacentCollapser.doQuery(null, null, null); + nonAdjacentCollapser.doCollapsing(uncollapsedDocset, index); + + Map collapseCounts = (Map) nonAdjacentCollapser.collapseContext.get("documentHeadCount"); + assertEquals(1, collapseCounts.size()); + assertEquals(1, collapseCounts.get(2).getCount()); + + DocSet docHeadsDocSet = nonAdjacentCollapser.createDocumentCollapseResult().getCollapsedDocset(); + assertEquals(6, docHeadsDocSet.size()); + + boolean zeroFound = false; + boolean oneFound = false; + boolean twoFound = false; + boolean threeFound = false; + boolean fourFound = false; + boolean fiveFound = false; + boolean sixFound = false; + for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) { + int docId = i.nextDoc(); + if (docId == 0) { + zeroFound = true; + } else if (docId == 1) { + oneFound = true; + } else if (docId == 2) { + twoFound = true; + } else if (docId == 3) { + threeFound = true; + } else if (docId == 4) { + fourFound = true; + } else if (docId == 5) { + fiveFound = true; + } else if (docId == 6) { + sixFound = true; + } + } + + assertTrue(zeroFound); + assertTrue(oneFound); + assertTrue(twoFound); + assertTrue(threeFound); + assertTrue(fourFound); + assertFalse(fiveFound); + assertTrue(sixFound); + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/DocumentCollapser.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/DocumentCollapser.java Fri Oct 09 13:40:41 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/DocumentCollapser.java Fri Oct 09 13:40:41 CEST 2009 @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; + +import java.io.IOException; +import java.util.List; + +/** + * A document collapser is responsible for removing documents that have the same value on a predefined field, + * only the most relevant (based sorting and / or scoring) documents will not be removed.
+ *
+ * Besides that a document collapser also creates statistics about the collaped docset e.g. how many documents + * were collapsed under the most relevant document and how many documents were collaped under a certain field value. + */ +public interface DocumentCollapser { + + /** + * Executes a search based on the specified parameters + * Returns a docset only containing the non collapsed documents also known as the document heads. + * + * @param mainQuery The main query + * @param filterQueries The filter queries + * @param sort The sort + * @return a docset only containing the non collapsed documents + * @throws IOException if index searcher related problems occur + */ + DocumentCollapseResult collapse(Query mainQuery, List filterQueries, Sort sort) throws IOException; + + /** + * Returns the statistics about the collapsed DocSet. + * The following information is returned: + *

+ * + * This method may only be invoked when the {@link this#collapse(Query, List, Sort)} has been invoked. + * + * @param searcher The solr index searcher + * @param docs The doclist containing the results to be displayed + * @return collapse counts for all documents in the specified docList. + * @throws IOException if index searcher related problems occur + */ + NamedList getCollapseInfo(SolrIndexSearcher searcher, DocList docs) throws IOException; + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/SumFunction.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/SumFunction.java Sun Oct 18 23:40:34 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/SumFunction.java Sun Oct 18 23:40:34 CEST 2009 @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector.aggregate; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +import java.math.BigDecimal; +import java.util.HashMap; +import java.util.Map; + +/** + * Computes the sum of all nummeric values per collapse group. + */ +public class SumFunction implements AggregateFunction { + + private final Map numbers = new HashMap(); + + public SumFunction() { + } + + public void collect(CollapseGroup collapseGroup, String value) { + BigDecimal number = numbers.get(collapseGroup); + if (number == null) { + numbers.put(collapseGroup, new BigDecimal(value)); + } else { + number = number.add(new BigDecimal(value)); + numbers.put(collapseGroup, number); + } + } + + public String calculate(CollapseGroup collapseGroup) { + Number number = numbers.get(collapseGroup); + return number != null ? number.toString() : null; + } + + public String getName() { + return "sum"; + } +} Index: src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java =================================================================== --- src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java (revision 816372) +++ src/solrj/org/apache/solr/client/solrj/response/QueryResponse.java Fri Oct 09 13:40:41 CEST 2009 @@ -17,21 +17,16 @@ package org.apache.solr.client.solrj.response; -import java.util.ArrayList; -import java.util.Date; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.NamedList; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.beans.DocumentObjectBinder; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; +import java.util.*; + /** * - * @version $Id: QueryResponse.java 816372 2009-09-17 20:28:53Z yonik $ + * @version $Id: QueryResponse.java 763791 2009-04-09 20:24:34Z ryan $ * @since solr 1.3 */ @SuppressWarnings("unchecked") @@ -46,6 +41,7 @@ private NamedList _highlightingInfo = null; private NamedList _spellInfo = null; private NamedList _statsInfo = null; + private NamedList _collapseInfo = null; // Facet stuff private Map _facetQuery = null; @@ -59,6 +55,9 @@ // SpellCheck Response private SpellCheckResponse _spellResponse = null; + // Field collapse response + private FieldCollapseResponse _fieldCollapseResponse = null; + // Field stats Response private Map _fieldStatsInfo = null; @@ -118,9 +117,17 @@ _statsInfo = (NamedList) res.getVal( i ); extractStatsInfo( _statsInfo ); } + else if ("collapse_counts".equals(n)) { + _collapseInfo = (NamedList) res.getVal(i); + extractFieldCollapseInfo(_collapseInfo); - } - } + } + } + } + private void extractFieldCollapseInfo(NamedList collapseInfo) { + _fieldCollapseResponse = new FieldCollapseResponse(collapseInfo); + } + private void extractSpellCheckInfo(NamedList spellInfo) { _spellResponse = new SpellCheckResponse(spellInfo); } @@ -178,11 +185,9 @@ // Parse the queries _facetQuery = new HashMap(); NamedList fq = (NamedList) info.get( "facet_queries" ); - if (fq != null) { - for( Map.Entry entry : fq ) { - _facetQuery.put( entry.getKey(), entry.getValue() ); - } + for( Map.Entry entry : fq ) { + _facetQuery.put( entry.getKey(), entry.getValue() ); + } - } // Parse the facet info into fields // TODO?? The list could be or ? If always then we can switch to @@ -276,6 +281,10 @@ return _spellResponse; } + public FieldCollapseResponse getFieldCollapseResponse() { + return _fieldCollapseResponse; + } + /** * See also: {@link #getLimitingFacets()} */ Index: src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AverageFunction.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AverageFunction.java Tue Oct 20 16:07:30 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AverageFunction.java Tue Oct 20 16:07:30 CEST 2009 @@ -0,0 +1,64 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector.aggregate; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; +import org.apache.solr.search.fieldcollapse.util.Counter; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.HashMap; +import java.util.Map; + +/** + * Computes the average (Arithmetic mean) of per collapse group. + */ +public class AverageFunction implements AggregateFunction { + + private final Map sums = new HashMap(); + private final Map times = new HashMap(); + + public void collect(CollapseGroup collapseGroup, String value) { + BigDecimal number = sums.get(collapseGroup); + if (number == null) { + sums.put(collapseGroup, new BigDecimal(value)); + times.put(collapseGroup, new Counter(1)); + } else { + number = number.add(new BigDecimal(value)); + sums.put(collapseGroup, number); + times.get(collapseGroup).increment(); + } + } + + public String calculate(CollapseGroup collapseGroup) { + BigDecimal number = sums.get(collapseGroup); + if (number == null) { + return null; + } + + int nrOfOccurences = times.get(collapseGroup).getCount(); + // TODO: need to find a way to specify the scale via the request + BigDecimal result = number.divide(new BigDecimal(nrOfOccurences), 2, RoundingMode.HALF_UP); + return result.toString(); + } + + public String getName() { + return "avg"; + } + +} Index: src/java/org/apache/solr/handler/component/CollapseComponent.java =================================================================== --- src/java/org/apache/solr/handler/component/CollapseComponent.java Sun Oct 25 20:38:36 CET 2009 +++ src/java/org/apache/solr/handler/component/CollapseComponent.java Sun Oct 25 20:38:36 CET 2009 @@ -0,0 +1,253 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.search.DocListAndSet; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.fieldcollapse.AdjacentDocumentCollapser; +import org.apache.solr.search.fieldcollapse.DocumentCollapseResult; +import org.apache.solr.search.fieldcollapse.DocumentCollapser; +import org.apache.solr.search.fieldcollapse.NonAdjacentDocumentCollapser; +import org.apache.solr.search.fieldcollapse.collector.*; +import org.apache.solr.util.plugin.SolrCoreAware; + +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Collapse component is responsible for do field collapsing with an implementation of {@link org.apache.solr.search.fieldcollapse.DocumentCollapser}, + * that does the most of the work. Collapsing is activated by specifying collapse.field in the request + * as parameter. + * If this parameter is not specified it falls back to the {@link QueryComponent#process(ResponseBuilder)} method. + *

+ * If the parameter collapse.facet with value after is specified, + * it replaces the collapsed docSet from DocListAndSet with the uncollapsed docset, so that facetation + * counts are based on the uncollapsed search result (which is usually desired behaviour) + * + * @version $Id: CollapseComponent.java 602341 2007-12-08 07:27:49Z ryan $ + * @since solr 1.3 + */ +public class CollapseComponent extends QueryComponent implements SolrCoreAware { + + private List collapseCollectorFactories = new ArrayList(); + private List collapseCollectorFactoryNames; + + /** + * {@inheritDoc} + */ + @Override + @SuppressWarnings("unchecked") + public void init(NamedList args) { + super.init(args); + collapseCollectorFactoryNames = (List) args.get("collapseCollectorFactories"); + } + + /** + * {@inheritDoc} + */ + public void inform(SolrCore core) { + if (collapseCollectorFactoryNames == null) { + collapseCollectorFactories.add(new DocumentGroupCountCollapseCollectorFactory()); + collapseCollectorFactories.add(new FieldValueCountCollapseCollectorFactory()); + collapseCollectorFactories.add(new DocumentFieldsCollapseCollectorFactory()); + // ok is not have its done normally, but works for now... need something like loadSearchComponents() and addIfNotPresent() + // but requires more changes outside field collapse code. + AggregateCollapseCollectorFactory aggregateCollapseCollectorFactory = new AggregateCollapseCollectorFactory(); + aggregateCollapseCollectorFactory.init(new NamedList()); + collapseCollectorFactories.add(aggregateCollapseCollectorFactory); + return; + } + + Map registry = new HashMap(); + core.initPlugins(registry, CollapseCollectorFactory.class); + for (String name : collapseCollectorFactoryNames) { + CollapseCollectorFactory factory = registry.get(name); + if (factory == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown collapse collector factory: " + name); + } + + collapseCollectorFactories.add(factory); + } + } + + /** + * Parses some parameters from the request in preparation for the collapsing when collapsing is enabled otherwise + * behaves like {@link super#prepare(ResponseBuilder)}. + * + * @param rb The response builder of the current search request + * @throws IOException When index search problems occur + */ + @Override + public void prepare(ResponseBuilder rb) throws IOException { + super.prepare(rb); + rb.req.getContext().put("collapseRequest", resolveCollapseRequest(rb)); + } + + /** + * Actually run the query + */ + @Override + public void process(ResponseBuilder rb) throws IOException { + CollapseRequest collapseRequest = (CollapseRequest) rb.req.getContext().remove("collapseRequest"); + if (collapseRequest == null) { + super.process(rb); + return; + } + doProcess(rb, collapseRequest); + } + + + // ================================================= Helpers ========================================================= + + /** + * Resolves the field collapse parameters from the request and constructs a CollapseRequest from it. + * + * @param rb The response builder that contains the request + * @return a CollapseRequest based on the parameters in the request + * @throws IOException When IO related problems occur + */ + protected CollapseRequest resolveCollapseRequest(ResponseBuilder rb) throws IOException { + SolrParams params = rb.req.getParams(); + if (params.get(CollapseParams.COLLAPSE_FIELD) == null) { + return null; + } + + String facet = params.get(CollapseParams.COLLAPSE_FACET); + CollapseParams.CollapseFacet collapseFacet = (facet != null) ? CollapseParams.CollapseFacet.get(facet) : CollapseParams.CollapseFacet.AFTER; + + String type = params.get(CollapseParams.COLLAPSE_TYPE); + CollapseParams.CollapseType collapseType = (type != null) ? CollapseParams.CollapseType.get(type) : CollapseParams.CollapseType.NORMAL; + DocumentCollapser documentCollapser = constructCollapser(rb, collapseType); + return new CollapseRequest(documentCollapser, collapseFacet); + } + + /** + * Executes a search and collapses the search results. + * + * @param rb The response builder of the current search request + * @param collapseRequest The current collapse request + * @throws IOException When index search problems occur + */ + protected void doProcess(ResponseBuilder rb, CollapseComponent.CollapseRequest collapseRequest) throws IOException { + SolrQueryResponse rsp = rb.rsp; + SolrQueryRequest req = rb.req; + SolrIndexSearcher searcher = req.getSearcher(); + DocumentCollapseResult collapseResult = + collapseRequest.getCollapser().collapse(rb.getQuery(), rb.getFilters(), rb.getSortSpec().getSort()); + + DocListAndSet results = searcher.getDocListAndSet(rb.getQuery(), + collapseResult == null ? rb.getFilters() : null, + collapseResult.getCollapsedDocset(), + rb.getSortSpec().getSort(), + rb.getSortSpec().getOffset(), + rb.getSortSpec().getCount(), + rb.getFieldFlags()); + + //for getting the facet count BEFORE the collapsing, we must + //get the doc. collection without filtering by the collapseResult. + boolean facetAfterCollapse = (collapseRequest.getCollapseFacet() == CollapseParams.CollapseFacet.AFTER); + if (!facetAfterCollapse) { + results.docSet = collapseResult.getUnCollapsedDocset(); + } + + rb.setResults(results); + rsp.add("collapse_counts", collapseRequest.getCollapser().getCollapseInfo(searcher, results.docList)); + rsp.add("response", results.docList); + } + + /** + * Returns the field collapser implementation based on the specified collapseType + * + * @param rb The response builder of the current search request + * @param collapseType The type of collapsing for the current search + * @return collapser implementation + * @throws IOException When index search problems occur + */ + protected DocumentCollapser constructCollapser(ResponseBuilder rb, CollapseParams.CollapseType collapseType) throws IOException { + switch (collapseType) { + case ADJACENT: + return new AdjacentDocumentCollapser(rb, collapseCollectorFactories); + case NORMAL: + return new NonAdjacentDocumentCollapser(rb, collapseCollectorFactories); + default: + throw new UnsupportedOperationException("Method not implemented"); + } + } + + ///////////////////////////////////////////// + /// SolrInfoMBean + //////////////////////////////////////////// + + @Override + public String getDescription() { + return "Field Collapsing"; + } + + @Override + public String getVersion() { + return ""; + } + + @Override + public String getSourceId() { + return ""; + } + + @Override + public String getSource() { + return ""; + } + + @Override + public URL[] getDocs() { + return null; + } + + + // ============================================ Inner Classes ======================================================== + + static class CollapseRequest { + + private final DocumentCollapser collapser; + private final CollapseParams.CollapseFacet collapseFacet; + + private CollapseRequest(DocumentCollapser collapser, CollapseParams.CollapseFacet collapseFacet) { + this.collapser = collapser; + this.collapseFacet = collapseFacet; + } + + public DocumentCollapser getCollapser() { + return collapser; + } + + public CollapseParams.CollapseFacet getCollapseFacet() { + return collapseFacet; + } + + } +} Index: src/solrj/org/apache/solr/client/solrj/SolrQuery.java =================================================================== --- src/solrj/org/apache/solr/client/solrj/SolrQuery.java (revision 823653) +++ src/solrj/org/apache/solr/client/solrj/SolrQuery.java Tue Oct 13 21:48:16 CEST 2009 @@ -17,11 +17,7 @@ package org.apache.solr.client.solrj; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.HighlightParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.StatsParams; +import org.apache.solr.common.params.*; /** @@ -571,6 +567,42 @@ return this.getInt(CommonParams.TIME_ALLOWED); } + /** + * Enables field collapsing for the current query with the specified field. + * + * @param field The field to collapse on + * @return the updated SolrQuery + */ + public SolrQuery enableFieldCollapsing(String field) { + add(CollapseParams.COLLAPSE_FIELD, field); + return this; + } + + /** + * Enables the inclusion of collapsed documents in the response. The fields parameter specify what fields are returned + * from the collasped documents. The fewer fields to return the better the performance is. When the argument specified + * is null or empty all fields will be returned. + * + * @param fields The fields to return for collapsed documents. If null or empty all fields are returned. + * @return the updated SolrQuery + */ + public SolrQuery includeCollapsedDocuments(String... fields) { + if (fields == null || fields.length < 1) { + add(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, "*"); + return this; + } + + StringBuilder fl = new StringBuilder(); + for (int i = 0; i < fields.length; i++) { + fl.append(fields[i]); + if (i + 1 < fields.length) { + fl.append(','); + } + } + add(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, fl.toString()); + return this; + } + /////////////////////// // Utility functions /////////////////////// Index: src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java =================================================================== --- src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java Fri Oct 09 13:40:42 CEST 2009 +++ src/test/org/apache/solr/client/solrj/response/FieldCollapseResponseTest.java Fri Oct 09 13:40:42 CEST 2009 @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.response; + +import junit.framework.TestCase; +import org.apache.solr.client.solrj.ResponseParser; +import org.apache.solr.client.solrj.impl.XMLResponseParser; +import org.apache.solr.common.util.NamedList; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; + +/** + * Tests for {@link org.apache.solr.client.solrj.response.FieldCollapseResponse} + */ +public class FieldCollapseResponseTest extends TestCase { + + /* + * Pass condition: expected values in test file must appear in FieldCollapseResponse + */ + public void testFieldCollapseResponse() throws Exception { + InputStream testDataStream = new FileInputStream(new File("fieldcollapse/testResponse.xml")); + ResponseParser responseParser = new XMLResponseParser(); + NamedList response = responseParser.processResponse(new InputStreamReader(testDataStream, "UTF-8")); + + QueryResponse queryResponse = new QueryResponse(); + queryResponse.setResponse(response); + FieldCollapseResponse result = queryResponse.getFieldCollapseResponse(); + + assertNotNull(result); + assertEquals("venue", result.getCollapseField()); + + assertEquals(1, result.getFieldValueCollapseCounts().size()); + assertEquals("melkweg", result.getFieldValueCollapseCounts().get(0).getFieldValue()); + assertEquals(1, result.getFieldValueCollapseCounts().get(0).getCount()); + + assertEquals(1, result.getDocumentIdCollapseCounts().size()); + assertEquals(233238L, result.getDocumentIdCollapseCounts().get(0).getDocumentId()); + assertEquals(1, result.getDocumentIdCollapseCounts().get(0).getCount()); + + assertEquals(1, result.getCollapsedDocuments().size()); + assertEquals(1, result.getCollapsedDocuments().get("melkweg").getNumFound()); + assertEquals("Amsterdam", result.getCollapsedDocuments().get("melkweg").get(0).getFieldValue("city")); + } +} Index: src/java/org/apache/solr/util/DocSetScoreCollector.java =================================================================== --- src/java/org/apache/solr/util/DocSetScoreCollector.java Fri Oct 09 13:40:42 CEST 2009 +++ src/java/org/apache/solr/util/DocSetScoreCollector.java Fri Oct 09 13:40:42 CEST 2009 @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.util; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.util.OpenBitSet; +import org.apache.solr.search.BitDocSet; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.DocSetAwareCollector; + +import java.io.IOException; + +/** + * Collects the documents with the score. + */ +public class DocSetScoreCollector extends DocSetAwareCollector { + private final float[] scores; + private final OpenBitSet bits; + + private Scorer scorer = null; + private int docBase = 0; + + /** + * Constructs a DocSetHitCollector with the specified parameter. + * + * @param maxDoc The maximum size of all the documents in the index + */ + public DocSetScoreCollector(int maxDoc) { + scores = new float[maxDoc]; + bits = new OpenBitSet(maxDoc); + } + + /** + * Collects documents as specified in {@link super#collect(int)}. + * Also stores the score associated with this document. + * + * @param doc The document id to collect + * @throws IOException + */ + public void collect(int doc) throws IOException { + doc = doc + docBase; + bits.fastSet(doc); + scores[doc] = scorer.score(); + } + + /** + * {@inheritDoc} + */ + public boolean acceptsDocsOutOfOrder() { + return false; + } + + // ================================================= Setter/Getter =================================================== + + /** + * {@inheritDoc} + */ + public void setNextReader(IndexReader reader, int docBase) throws IOException { + this.docBase = docBase; + } + + /** + * {@inheritDoc} + */ + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + + /** + * Returns a docset of the collected documents. + * + * @return a docset of the collected documents + */ + public DocSet getDocSet() { + return new BitDocSet(bits); + } + + /** + * Returns the scores for the collected documents. + * The index of the score represents the lucene document identifier. + * + * @return the scores for the collected documents + */ + public float[] getScores() { + return scores; + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapser.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapser.java Sun Oct 25 15:42:09 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/NonAdjacentDocumentCollapser.java Sun Oct 25 15:42:09 CET 2009 @@ -0,0 +1,495 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.*; +import org.apache.lucene.util.PriorityQueue; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollector; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; +import org.apache.solr.util.DocSetScoreCollector; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Normal collapsing behaviour. Collapses all documents that have the same field value on the predefined field + * and that have occured equal or more then collapse threshold. + */ +public class NonAdjacentDocumentCollapser extends AbstractDocumentCollapser { + + DocumentComparator documentComparator; + + /** + * See {@link super#AbstractDocumentCollapser(org.apache.solr.handler.component.ResponseBuilder, List)} + */ + public NonAdjacentDocumentCollapser(ResponseBuilder rb, List collapseCollectorFactories) throws IOException { + super(rb, collapseCollectorFactories); + } + + /** + * {@inheritDoc} + */ + @Override + protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException { + final int maxDoc = searcher.maxDoc(); + if (containsSortOnScore(sort)) { + DocSetScoreCollector docSetCollector = new DocSetScoreCollector(searcher.maxDoc()); + long startTime = System.currentTimeMillis(); + DocSet filter = searcher.getDocSet(filterQueries); + uncollapsedDocSet =searcher.getDocSet(mainQuery, filter, docSetCollector); + timeCreateUncollapedDocset = System.currentTimeMillis() - startTime; + if (sort == null) { + sort = new Sort(new SortField("score", SortField.SCORE, true)); + } + + documentComparator = new DocumentComparator(sort, maxDoc, searcher.getIndexReader(), docSetCollector.getScores()); + } else { + long startTime = System.currentTimeMillis(); + DocSet filter = searcher.getDocSet(filterQueries); + uncollapsedDocSet = searcher.getDocSet(mainQuery, filter); + timeCreateUncollapedDocset = System.currentTimeMillis() - startTime; + + documentComparator = new DocumentComparator(sort, maxDoc, searcher.getIndexReader()); + } + } + + /** + * {@inheritDoc} + */ + protected void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values) { + long startTime = System.currentTimeMillis(); + int docCount = 0; + // Keep how many documents we have processed the track of how many docs + // with the same collapse value we have processed so far. + Map collapsedDocs = new HashMap(); + + for (DocIterator i = uncollapsedDocset.iterator(); i.hasNext();) { + int currentId = i.nextDoc(); + String currentValue = values.lookup[values.order[currentId]]; + + // Get the last doc. and the total amount of docs. we have seen so + // far for this collapsing value + CollapsedDocumentGroup collapseDoc = collapsedDocs.get(currentValue); + if (collapseDoc == null) { + // new collapsing value => create a new record for it + collapseDoc = new CollapsedDocumentGroup(0, 0, documentComparator, collapseThreshold); + collapsedDocs.put(currentValue, collapseDoc); + } + // dropoutId has a value smaller than the smallest value in the queue and therefore it was removed from the queue + Integer dropOutId = (Integer) collapseDoc.priorityQueue.insertWithOverflow(currentId); + + // check if we have reached the collapse threshold, if so start counting collapsed documents + if (++collapseDoc.totalCount > collapseThreshold) { + collapseDoc.collapsedDocuments++; + if (dropOutId != null) { + NonAdjacentCollapseGroup collapseValue = new NonAdjacentCollapseGroup(currentValue); + for (CollapseCollector collector : collectors) { + collector.documentCollapsed(dropOutId, collapseValue, collapseContext); + } + } + } + + // Stop after collapseMaxDocs documents + if (++docCount >= collapseMaxDocs) { + break; + } + } + + // adding the head documents to the internal document buffer and + // adding the collapsed counts per document head to the map + for (String fieldValue : collapsedDocs.keySet()) { + CollapsedDocumentGroup collapseDoc = collapsedDocs.get(fieldValue); + if (collapseDoc.collapsedDocuments > 0) { + NonAdjacentCollapseGroup collapseValue = new NonAdjacentCollapseGroup(fieldValue); + for (CollapseCollector collector : collectors) { + collector.documentHead((Integer) collapseDoc.priorityQueue.top(), collapseValue, collapseContext); + } + } + Integer doc; + while ((doc = (Integer) collapseDoc.priorityQueue.pop()) != null) { + addDoc(doc); + } + } + + timeCollapsing = System.currentTimeMillis() - startTime; + } + + + // ================================================= Helpers ========================================================= + + /** + * Returns true if the sort contains a sortfield that sorts on score, otherwise false. + * + * @param sort The sort + * @return true if the sort contains a sortfield that sorts on score, otherwise false + */ + protected boolean containsSortOnScore(Sort sort) { + if (sort == null) { + return true; // means default sorting, which is sorting on score desc + } + + for (SortField field : sort.getSort()) { + if (field.getType() == SortField.SCORE) { + return true; + } + } + return false; + } + + + // ============================================ Inner Classes ======================================================== + + private static class NonAdjacentCollapseGroup implements CollapseGroup { + + private final String fieldValue; + + private NonAdjacentCollapseGroup(String fieldValue) { + this.fieldValue = fieldValue; + } + + public String getKey() { + return fieldValue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + NonAdjacentCollapseGroup that = (NonAdjacentCollapseGroup) o; + + if (fieldValue != null ? !fieldValue.equals(that.fieldValue) : that.fieldValue != null) return false; + + return true; + } + + @Override + public int hashCode() { + return fieldValue != null ? fieldValue.hashCode() : 0; + } + } + + /** + * A PriorityQueue that maintaince order with a DocumentComparator. + */ + public static class DocumentPriorityQueue extends PriorityQueue { + + private final DocumentComparator comparator; + + /** + * Constructs a DocumentPriorityQueue + * + * @param comparator The DocumentComparator used for maintaining order in the queue + * @param max The maximum number of document identifiers in the queue (is equal to collapse thresold paramter) + */ + public DocumentPriorityQueue(DocumentComparator comparator, int max) { + this.comparator = comparator; + initialize(max); + } + + /** + * {@inheritDoc} + */ + protected boolean lessThan(Object a, Object b) { + return comparator.compare((Integer) a, (Integer) b) < 0; + } + } + + /** + * Compares two documents with each other. + */ + public static class DocumentComparator { + + private final FieldComparator[] fieldComparators; + private final boolean[] descending; + private Scorer scorer; + + /** + * Constructs a DocumentComparator by initializing the + * {@link org.apache.lucene.search.FieldComparator}s and determining the sort orders. + * + * @param sort The sort used for the creation of the FieldComparators. + * @param numberOfHits The number of results in the pre-field-collapsed resultset + * @param reader The index reader, used for reading field values (in the FieldComparators) + */ + public DocumentComparator(Sort sort, int numberOfHits, IndexReader reader) { + fieldComparators = new FieldComparator[sort.getSort().length]; + descending = new boolean[sort.getSort().length]; + initializeFieldComparators(sort, numberOfHits, reader, null); + } + + /** + * Constructs a DocumentComparator by initializing the + * {@link org.apache.lucene.search.FieldComparator}s and determining the sort orders. + * + * @param sort The sort used for the creation of the FieldComparators. + * @param numberOfHits The number of results in the pre-field-collapsed resultset + * @param reader The index reader, used for reading field values (in the FieldComparators) + * @param scores The scores used for comparing the documents + */ + public DocumentComparator(Sort sort, int numberOfHits, IndexReader reader, float[] scores) { + fieldComparators = new FieldComparator[sort.getSort().length]; + descending = new boolean[sort.getSort().length]; + initializeFieldComparators(sort, numberOfHits, reader, scores); + } + + /** + * Compares doc1 and doc2 with each other. + * Compares the two documents with the initialized FieldComparators, + * if all FieldComparators compares the documents as equal then the document with lowest lucene + * identifier will be classified as most relevant. + * + * @param doc1 The lucene identifier of the first document + * @param doc2 The lucene identifier of the second document + * @return -1 if doc1 is less relevant or equal relevant but has a higher lucene id then doc2, + * 0 if both documents are identical + * 1 if doc1 is more relevant or equal relevant but has a lower lucene id then doc2 + */ + public int compare(int doc1, int doc2) { + int result; + for (int i = 0; i < fieldComparators.length; i++) { + FieldComparator fieldComparator = fieldComparators[i]; + try { + scorer.advance(doc1); + fieldComparator.copy(doc1, doc1); + scorer.advance(doc2); + fieldComparator.copy(doc2, doc2); + } catch (IOException e) { + throw new RuntimeException(e); + } + result = fieldComparator.compare(doc1, doc2); + result = descending[i] ? result : -result; + + if (result != 0) { + return result; + } + } + + // field comparators identified the field(s) values as equal + // Document with lowest identifier has higher precendence + if (doc1 < doc2) { + return 1; + } else if (doc1 > doc2) { + return -1; + } + + // can only happen if comparing two the exact same document (with same lucene id) + return 0; + } + + private void initializeFieldComparators(Sort sort, int numberOfHits, IndexReader indexReader, float[] scores) { + try { + scorer = new PredefinedScorer(scores == null ? new float[0] : scores); + if (sort.getSort().length == 1 && scores != null) { + fieldComparators[0] = new FloatValueFieldComparator(scores); + descending[0] = sort.getSort()[0].getReverse(); + return; + } + + for (int i = 0; i < sort.getSort().length; i++) { + SortField sortField = sort.getSort()[i]; + fieldComparators[i] = sortField.getComparator(numberOfHits, i); + descending[i] = sortField.getReverse(); + fieldComparators[i].setNextReader(indexReader, 0); + fieldComparators[i].setScorer(scorer); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + } + + /** + * Represents a collapse group, that collects statistics for a certain fieldvalue group. + *

+ * Keeps track of the following statistics during the collapsing on fieldvalue: + *

    + *
  • how many documents were collapsed + *
  • what the most relevant head documents are in this group (these will not get collapsed) + *
  • how many have been procesed + *
+ */ + private static class CollapsedDocumentGroup { + + int collapsedDocuments; + int totalCount; + final DocumentPriorityQueue priorityQueue; + + /** + * Constructs a CollapsedDocumentGroup. Keeps track of the of the most relevant documents + * in this group. These documents will stay in the resultset and do not get collaped, no more then the + * specified collpasThreshold will be kept inside this CollapsedDocumentGroup. + * + * @param totalCount the total amount documents processed in the collapsing process + * @param collapsedDocuments the amount of documents collapsed under this group + * @param comparator The document comparater used inside priority queue + * @param collapsThreshold The threshold to start collapsing from + */ + private CollapsedDocumentGroup(int totalCount, int collapsedDocuments, DocumentComparator comparator, int collapsThreshold) { + this.totalCount = totalCount; + this.collapsedDocuments = collapsedDocuments; + this.priorityQueue = new DocumentPriorityQueue(comparator, collapsThreshold); + } + } + + /** + * A scorer that returns scores from a predefined array of scores. + */ + private static final class PredefinedScorer extends Scorer { + + private final float[] scores; + private int index; + + private PredefinedScorer(float[] scores) { + super(null); + this.scores = scores; + } + + /** + * {@inheritDoc} + */ + public float score() throws IOException { + return scores[index]; + } + + /** + * {@inheritDoc} + */ + public int docID() { + return index; + } + + /** + * {@inheritDoc} + */ + public int nextDoc() throws IOException { + return index < scores.length ? ++index : NO_MORE_DOCS; + } + + /** + * {@inheritDoc} + */ + public int advance(int target) throws IOException { + if (target < scores.length) { + index = target; + return index; + } + return NO_MORE_DOCS; + } + + } + + /** + * Field comparator for floats. This implementation is a little bit faster (around 30 ms) then {@link FloatComparator.FloatComparator} + * because it does not copy floats in {@link #copy(int, int)}. This is not necessary in fieldcollapsing case, because + * the arguments passed to the {@link #compare(int, int)} method are document ids and the values array is sorted on + * document order. + */ + private static class FloatValueFieldComparator extends FieldComparator { + + private final float[] values; + + private FloatValueFieldComparator(float[] values) { + this.values = values; + } + + /** + * Compares value for doc1 with value for doc2. + * + * @param doc1 first document to compare + * @param doc2 second document to compare + * @return any N < 0 if doc2's value is sorted after + * doc1, any N > 0 if the doc2's value is sorted before + * doc1 and 0 if they are equal + */ + public int compare(int doc1, int doc2) { + final float value1 = values[doc1]; + final float value2 = values[doc2]; + + if (value1 > value2) { + return 1; + } else if (value1 < value2) { + return -1; + } else { + return 0; + } + } + + /** + * Unsupported, is not necessary for field collapsing + * + * @param doc The document identifier + * @throws UnsupportedOperationException + */ + public void setBottom(int doc) { + throw new UnsupportedOperationException("Method not implemented"); + } + + /** + * Unsupported, is not necessary for field collapsing + * + * @param doc The document identifier + * @return nothing + * @throws UnsupportedOperationException + */ + public int compareBottom(int doc) throws IOException { + throw new UnsupportedOperationException("Method not implemented"); + } + + /** + * {@inheritDoc} + *

+ * Does nothing, necessary to implement because of superclass. + */ + public void copy(int slot, int doc) throws IOException { + } + + /** + * {@inheritDoc} + *

+ * Does nothing, necessary to implement because of superclass. + */ + public void setNextReader(IndexReader reader, int docBase) throws IOException { + } + + /** + * {@inheritDoc} + */ + public int sortType() { + return SortField.FLOAT; + } + + /** + * Unsupported, is not necessary for field collapsing + * + * @param doc The document identifier + * @throws UnsupportedOperationException + */ + public Comparable value(int doc) { + throw new UnsupportedOperationException("Method not implemented"); + } + + } +} Index: src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java =================================================================== --- src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java Fri Oct 09 13:40:41 CEST 2009 +++ src/solrj/org/apache/solr/client/solrj/response/FieldCollapseResponse.java Fri Oct 09 13:40:41 CEST 2009 @@ -0,0 +1,219 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.client.solrj.response; + +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Represents the fieldcollapse part of the response. + * This class encapsulates the following information: + *
    + *
  • What field was used during field collapsing. + *
  • How many documents were collapsed under a certain field value + *
  • How many documents were collapsed under a group head + *
  • The collapsed documents per field value. + *
+ */ +public class FieldCollapseResponse { + + private final String collapseField; + private final List fieldValueCollapseCounts; + private final List documentIdCollapseCounts; + private final Map collapsedDocuments; + + /** + * Constructs a FieldCollapseResponse instance. + * + * @param collapseInfo The field collapse part of the response. From the namelist collapse_counts + */ + public FieldCollapseResponse(NamedList collapseInfo) { + this.collapseField = (String) collapseInfo.get("field"); + this.fieldValueCollapseCounts = parseFieldValueCollapseCounts((NamedList) collapseInfo.get("count")); + this.documentIdCollapseCounts = parseDocumentIdCollapseCounts((NamedList) collapseInfo.get("doc")); + this.collapsedDocuments = parseCollapsedDocuments((NamedList) collapseInfo.get("collapsedDocs")); + } + + /** + * Returns the field name that was used during collapsing. + * + * @return the field name that was used during collapsing + */ + public String getCollapseField() { + return collapseField; + } + + /** + * Returns a list of {@link FieldValueCollapseCount}. That contains information about how many + * documents were collapsed under a certainer field value. + * + * @return a list of {@link FieldValueCollapseCount} + */ + public List getFieldValueCollapseCounts() { + return fieldValueCollapseCounts; + } + + /** + * Returns a list of {@link DocumentIdCollapseCount}, that contains information about how many + * documents were collapsed under a certain group head. + * + * @return a list of {@link DocumentIdCollapseCount} + */ + public List getDocumentIdCollapseCounts() { + return documentIdCollapseCounts; + } + + /** + * Returns all collapsed document as SolrDocumentList per field value in a map structure. + * + * Returns an empty map when no documents were collapsed or the inclusion of collapsed documents + * was not specified in the request + * + * @return all collapsed documents per field value + */ + public Map getCollapsedDocuments() { + return collapsedDocuments; + } + + + // ================================================ Helper Methods ================================================= + + private List parseFieldValueCollapseCounts(NamedList info) { + List collapseCounts = new ArrayList(); + + for (int i = 0; i < info.size(); i++) { + String fieldName = info.getName(i); + int count = info.getVal(i); + collapseCounts.add(new FieldValueCollapseCount(fieldName, count)); + } + + return collapseCounts; + } + + private List parseDocumentIdCollapseCounts(NamedList info) { + List collapseCounts = new ArrayList(); + + for (int i = 0; i < info.size(); i++) { + long documentId = Long.parseLong(info.getName(i)); + int count = info.getVal(i); + collapseCounts.add(new DocumentIdCollapseCount(documentId, count)); + } + + return collapseCounts; + } + + private Map parseCollapsedDocuments(NamedList info) { + Map collapsedDocuments = new HashMap(); + if (info == null) { + return collapsedDocuments; + } + + for (int i = 0; i < info.size(); i++) { + String fieldValue = info.getName(i); + SolrDocumentList documentList = info.getVal(i); + collapsedDocuments.put(fieldValue, documentList); + } + + return collapsedDocuments; + } + + + // ================================================= Inner classes ================================================= + + /** + * Represents how many documents where collapsed under a certain field value. + */ + public class FieldValueCollapseCount { + + private final String fieldValue; + private final int count; + + /** + * Constructs FieldValueCollapseCount. + * + * @param fieldValue The collapsed field value + * @param count The number of documents that collapsed under the specified field value. + */ + public FieldValueCollapseCount(String fieldValue, int count) { + this.fieldValue = fieldValue; + this.count = count; + } + + /** + * Returns the collapsed field value. + * + * @return the collapsed field value + */ + public String getFieldValue() { + return fieldValue; + } + + /** + * Returns the number of documents that collapsed under the specified field value. + * + * @return the number of documents that collapsed under the specified field value + */ + public int getCount() { + return count; + } + } + + /** + * Represents the head document id and how many documents were collapsed under it. + */ + public class DocumentIdCollapseCount { + + private final long documentId; + private final int count; + + /** + * Constructs DocumentIdCollapseCount. + * + * @param documentId The head document identifier + * @param count The number of documents that were collapsed under the head document + */ + public DocumentIdCollapseCount(long documentId, int count) { + this.documentId = documentId; + this.count = count; + } + + /** + * Returns the head document id. + * This is the most relevant document of the group (documents with a common field value) and thus was not collapsed. + * + * @return head document id + */ + public long getDocumentId() { + return documentId; + } + + /** + * Returns the number of documents that were collapsed under the document head. + * + * @return the number of documents that were collapsed under the document head + */ + public int getCount() { + return count; + } + } +} Index: src/java/org/apache/solr/search/fieldcollapse/util/Counter.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/util/Counter.java Tue Oct 20 16:39:57 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/util/Counter.java Tue Oct 20 16:39:57 CEST 2009 @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.util; + +/** + * A simple counter. In some parts of the field collapse code I need to keep track of the count. If I use an Integer for + * that I have the contant overhead of autoboxing and the creation of new Integer objects when I increment a counter. + * This class does not have that overhead because I increment the same variable. + *

+ * This class is not thread safe. + */ +public class Counter { + + private int count; + + /** + * Constructs a counter with count value zero. + */ + public Counter() { + this(0); + } + + /** + * Constructs a counter with the specified count. + * + * @param count The specified count + */ + public Counter(int count) { + this.count = count; + } + + /** + * Increments the counter. + */ + public void increment() { + count++; + } + + /** + * Sets the counter to the specified count + * + * @param count The specified count + */ + public void setCount(int count) { + this.count = count; + } + + /** + * Returns the current count of this counter. + * + * @return the current count + */ + public int getCount() { + return count; + } + +} Index: src/java/org/apache/solr/search/SolrIndexSearcher.java =================================================================== --- src/java/org/apache/solr/search/SolrIndexSearcher.java (revision 794328) +++ src/java/org/apache/solr/search/SolrIndexSearcher.java Wed Oct 14 22:23:31 CEST 2009 @@ -17,28 +17,31 @@ package org.apache.solr.search; -import org.apache.lucene.document.*; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.OpenBitSet; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrCore; import org.apache.solr.core.SolrInfoMBean; +import org.apache.solr.request.UnInvertedField; import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.SchemaField; -import org.apache.solr.request.UnInvertedField; -import org.apache.lucene.util.OpenBitSet; +import org.apache.solr.util.DocSetScoreCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.net.URL; import java.util.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** @@ -527,8 +530,33 @@ * This method can handle negative queries. *

* The DocSet returned should not be modified. + * + * @param query The specified query that must match with the document ids + * @return the set of document ids matching all queries. Set may not be modified + * @throws IOException If an IO related exception occurs */ public DocSet getDocSet(Query query) throws IOException { + return getDocSet(query, (DocSetAwareCollector) null); + } + + /** + * Returns the set of document ids matching a query. + * This method is cache-aware and attempts to retrieve the answer from the cache if possible. + * If the answer was not cached, it may have been inserted into the cache as a result of this call. + * This method can handle negative queries. + *

+ * The DocSet returned should not be modified. + * + *

+ * This method allows a custom collector the be specified to collect results in a custum way. For example to also + * collect the score of a collected document + * + * @param query The specified query that must match with the document ids + * @param collector A docset aware collector that collects the result + * @return the set of document ids matching all queries. Set may not be modified + * @throws IOException If an IO related exception occurs + */ + public DocSet getDocSet(Query query, DocSetAwareCollector collector) throws IOException { // Get the absolute value (positive version) of this query. If we // get back the same reference, we know it's positive. Query absQ = QueryUtils.getAbs(query); @@ -538,12 +566,12 @@ DocSet absAnswer = (DocSet)filterCache.get(absQ); if (absAnswer!=null) { if (positive) return absAnswer; - else return getPositiveDocSet(matchAllDocsQuery).andNot(absAnswer); + else return getPositiveDocSet(matchAllDocsQuery, collector).andNot(absAnswer); } } - DocSet absAnswer = getDocSetNC(absQ, null); - DocSet answer = positive ? absAnswer : getPositiveDocSet(matchAllDocsQuery).andNot(absAnswer); + DocSet absAnswer = getDocSetNC(absQ, null, collector); + DocSet answer = positive ? absAnswer : getPositiveDocSet(matchAllDocsQuery, collector).andNot(absAnswer); if (filterCache != null) { // cache negative queries as positive @@ -555,17 +583,21 @@ // only handle positive (non negative) queries DocSet getPositiveDocSet(Query q) throws IOException { + return getPositiveDocSet(q, null); + } + + // only handle positive (non negative) queries + DocSet getPositiveDocSet(Query q, DocSetAwareCollector collector) throws IOException { DocSet answer; if (filterCache != null) { answer = (DocSet)filterCache.get(q); if (answer!=null) return answer; } - answer = getDocSetNC(q,null); + answer = getDocSetNC(q, null, collector); if (filterCache != null) filterCache.put(q,answer); return answer; } - private static Query matchAllDocsQuery = new MatchAllDocsQuery(); /** @@ -575,6 +607,10 @@ * This method can handle negative queries. *

* The DocSet returned should not be modified. + * + * @param queries The specified list of queries that must match with the document ids + * @return the set of document ids matching all queries. Set may not be modified + * @throws IOException If an IO related exception occurs */ public DocSet getDocSet(List queries) throws IOException { if (queries==null) return null; @@ -624,10 +660,17 @@ // query must be positive protected DocSet getDocSetNC(Query query, DocSet filter) throws IOException { - DocSetCollector collector = new DocSetCollector(maxDoc()>>6, maxDoc()); + return getDocSetNC(query, filter, null); + } + // query must be positive + protected DocSet getDocSetNC(Query query, DocSet filter, DocSetAwareCollector collector) throws IOException { + if (collector == null) { + collector = new DocSetCollector(maxDoc()>>6, maxDoc()); + } + if (filter==null) { - if (query instanceof TermQuery) { + if (query instanceof TermQuery && !(collector instanceof DocSetScoreCollector)) { Term t = ((TermQuery)query).getTerm(); SolrIndexReader[] readers = reader.getLeafReaders(); int[] offsets = reader.getLeafOffsets(); @@ -659,7 +702,6 @@ } } - /** * Returns the set of document ids matching both the query and the filter. * This method is cache-aware and attempts to retrieve the answer from the cache if possible. @@ -671,8 +713,22 @@ * @return DocSet meeting the specified criteria, should not be modified by the caller. */ public DocSet getDocSet(Query query, DocSet filter) throws IOException { - if (filter==null) return getDocSet(query); + return getDocSet(query, filter, null); + } + /** + * Returns the set of document ids matching both the query and the filter. + * This method is cache-aware and attempts to retrieve the answer from the cache if possible. + * If the answer was not cached, it may have been inserted into the cache as a result of this call. + *

+ * + * @param query + * @param filter may be null + * @return DocSet meeting the specified criteria, should not be modified by the caller. + */ + public DocSet getDocSet(Query query, DocSet filter, DocSetAwareCollector collector) throws IOException { + if (filter==null) return getDocSet(query, collector); + // Negative query if absolute value different from original Query absQ = QueryUtils.getAbs(query); boolean positive = absQ==query; @@ -688,7 +744,7 @@ } // If there isn't a cache, then do a single filtered query if positive. - return positive ? getDocSetNC(absQ,filter) : filter.andNot(getPositiveDocSet(absQ)); + return positive ? getDocSetNC(absQ,filter,collector) : filter.andNot(getPositiveDocSet(absQ, collector)); } @@ -1196,6 +1252,43 @@ } /** + * Returns documents matching both query and the intersection + * of filterList, sorted by sort. + * Also returns the compete set of documents + * matching query and filter + * (regardless of offset and len). + *

+ * This method is cache aware and may retrieve filter from + * the cache or make an insertion into the cache as a result of this call. + *

+ * FUTURE: The returned DocList may be retrieved from a cache. + *

+ * The DocList and DocSet returned should not be modified. + * + * @param query The main query + * @param filterList may be null + * @param docSet filter docSet + * @param lsort criteria by which to sort (if null, query relevance is used) + * @param offset offset into the list of documents to return + * @param len maximum number of documents to return + * @param flags user supplied flags for the result set + * @return DocListAndSet meeting the specified criteria, should not be modified by the caller. + * @throws IOException If an IO related exception occurs + */ + public DocListAndSet getDocListAndSet(Query query, List filterList, DocSet docSet, Sort lsort, int offset, int len, int flags) throws IOException { + //DocListAndSet ret = new DocListAndSet(); + //getDocListC(ret,query,filterList,docSet,lsort,offset,len, flags |= GET_DOCSET); + + QueryCommand qc = new QueryCommand(); + qc.setQuery(query).setFilterList(filterList).setFilter(docSet); + qc.setSort(lsort).setOffset(offset).setLen(len).setFlags(flags |= GET_DOCSET); + QueryResult result = new QueryResult(); + getDocListC(result,qc); + + return result.getDocListAndSet(); + } + + /** * Returns documents matching both query and filter * and sorted by sort. Also returns the compete set of documents * matching query and filter (regardless of offset and len). Index: src/common/org/apache/solr/common/params/CollapseParams.java =================================================================== --- src/common/org/apache/solr/common/params/CollapseParams.java Sun Oct 11 19:12:45 CEST 2009 +++ src/common/org/apache/solr/common/params/CollapseParams.java Sun Oct 11 19:12:45 CEST 2009 @@ -0,0 +1,94 @@ +package org.apache.solr.common.params; + +import org.apache.solr.common.SolrException; + +public interface CollapseParams { + + /** + * The field to collapse results on. + */ + public static final String COLLAPSE_FIELD = "collapse.field"; + + /** + * Type of collapsing to perform: "normal" or "adjacent". + */ + public static final String COLLAPSE_TYPE = "collapse.type"; + + public enum CollapseType { + NORMAL, ADJACENT; + + public String toString() { + return super.toString().toLowerCase(); + } + + public static CollapseType get(String label) { + try { + return valueOf(label.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, label + + " is not a valid type of field collapsing", e); + } + } + } + + /** + * Apply faceting before or after collapsing. + */ + public static final String COLLAPSE_FACET = "collapse.facet"; + + public enum CollapseFacet { + BEFORE, AFTER; + + public String toString() { + return super.toString().toLowerCase(); + } + + public static CollapseFacet get(String label) { + try { + return valueOf(label.toUpperCase()); + } catch (IllegalArgumentException e) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, label + + " is not a valid faceting mode for field collapsing", e); + } + } + } + + /** + * The number of documents with the same value for collapse.field after which + * collapsing kicks in. + */ + public static final String COLLAPSE_THRESHOLD = "collapse.threshold"; + + /** + * @deprecated Deprecated in favour of collapse.threshold. + */ + public static final String COLLAPSE_MAX = "collapse.max"; + + /** + * Maximum number of documents to process during field collapsing. + */ + public static final String COLLAPSE_MAXDOCS = "collapse.maxdocs"; + + /** + * Return collapse count for each document? Defaults to true. + */ + public static final String COLLAPSE_INFO_DOC = "collapse.info.doc"; + + /** + * Return collapse count for each field value? Defaults to true. + */ + public static final String COLLAPSE_INFO_COUNT = "collapse.info.count"; + + /** + * Parameter indicating to return the collapsed documents in the response and what fields to return in comma separated manner. + * A value * indicates that all fields will be returned. + */ + public static final String COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS = "collapse.includeCollapsedDocs.fl"; + + /** + * Parameter indicating wheter to include collapse debug information + */ + public static final String COLLAPSE_DEBUG = "collapse.debug"; + + public static final String COLLAPSE_AGGREGATE = "collapse.aggregate"; +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollector.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollector.java Sun Oct 25 22:08:35 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/CollapseCollector.java Sun Oct 25 22:08:35 CET 2009 @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.solr.common.util.NamedList; +import org.apache.solr.search.DocList; +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +/** + * A CollapseCollector is responsible for receiving collapse callbacks from the DocumentCollapser. + * An implementation can choose what to do with the received callbacks and data. Whatever an implementation collects it + * is responsible for adding its results to the response. + * + * Implementation of this interface don't need to be thread safe! + */ +public interface CollapseCollector { + + /** + * Informs the CollapseCollector that a document has been collapsed under the specified collapseGroup. + * + * @param docId The id of the document that has been collasped + * @param collapseGroup The collapse group the docId has been collapsed under + * @param collapseContext The collapse context + */ + void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext); + + /** + * Informs the CollapseCollector about the document head. + * The document head is the most relevant id for the specified collapseGroup. + * + * @param docHeadId The identifier of the document head + * @param collapseGroup The collapse group of the document head + * @param collapseContext The collapse context + */ + void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext); + + /** + * Adds the CollapseCollector implementation specific result data to the result. + * + * @param result The response result + * @param docs The documents to be added to the response + * @param collapseContext The collapse context + */ + void getResult(NamedList result, DocList docs, CollapseContext collapseContext); + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/DocumentGroupCountCollapseCollectorFactory.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/DocumentGroupCountCollapseCollectorFactory.java Sun Oct 25 22:37:02 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/DocumentGroupCountCollapseCollectorFactory.java Sun Oct 25 22:37:02 CET 2009 @@ -0,0 +1,103 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.lucene.index.IndexReader; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.fieldcollapse.CollapseGroup; +import org.apache.solr.search.fieldcollapse.util.Counter; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * A collapse collector factory that creates collapse collectors that the collapse counts per document group and + * return the counts in the response per collapsed group most relevant document id. + * When this collapse collector factory is configured then this feature is enabled by default. + * To disable this for a search add collapse.info.doc=false in the request. + */ +public class DocumentGroupCountCollapseCollectorFactory implements CollapseCollectorFactory { + + /** + * {@inheritDoc} + */ + public CollapseCollector createCollapseCollector(SolrQueryRequest request) { + SolrParams params = request.getParams(); + boolean includeDocCount = params.getBool(CollapseParams.COLLAPSE_INFO_DOC, true); + return includeDocCount ? new DocumentCountCollapseCollector(request.getSearcher()) : null; + } + + + //============================================== Inner classes ======================================================= + + private static class DocumentCountCollapseCollector implements CollapseCollector { + + private final Map fieldValueCount = new HashMap(); + private final Map documentHeadCount = new HashMap(); + private final SolrIndexSearcher searcher; + + public DocumentCountCollapseCollector(SolrIndexSearcher searcher) { + this.searcher = searcher; + } + + public void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + Counter counter = fieldValueCount.get(collapseGroup); + if (counter == null) { + counter = new Counter(0); + fieldValueCount.put(collapseGroup, counter); + } + counter.increment(); + } + + public void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + documentHeadCount.put(docHeadId, fieldValueCount.get(collapseGroup)); + collapseContext.put("documentHeadCount", documentHeadCount); + } + + public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { + NamedList resDoc = new NamedList(); + IndexReader reader = searcher.getReader(); + String uniqueIdFieldname = searcher.getSchema().getUniqueKeyField().getName(); + + for (DocIterator i = docs.iterator(); i.hasNext();) { + int id = i.nextDoc(); + Counter counter = documentHeadCount.get(id); + if (counter == null) { + continue; + } + int count = counter.getCount(); + + try { + resDoc.add(reader.document(id).get(uniqueIdFieldname), count); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + result.add("doc", resDoc); + } + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/DocumentFieldsCollapseCollectorFactory.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/DocumentFieldsCollapseCollectorFactory.java Sun Oct 25 20:38:36 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/DocumentFieldsCollapseCollectorFactory.java Sun Oct 25 20:38:36 CET 2009 @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.util.OpenBitSet; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.fieldcollapse.CollapseGroup; +import org.apache.solr.update.DocumentBuilder; + +import java.io.IOException; +import java.util.*; +import java.util.regex.Pattern; + +/** + * A collector factory that creates collapse collectors that collect predefined fieldvalues from collapsed documents. + * In order to enable this specify collapse.includeCollapsedDocs.fl=[val] in the request. Where [val] may be a star (*) + * to include all field of the collapsed documents or a comma separated list of the collapsed documents' fieldnames to include. + * When configured this feature is not enabled by default. + */ +public class DocumentFieldsCollapseCollectorFactory implements CollapseCollectorFactory { + + private static final Pattern splitPattern = Pattern.compile(","); + + /** + * {@inheritDoc} + */ + public CollapseCollector createCollapseCollector(SolrQueryRequest request) { + SolrParams params = request.getParams(); + String collapsedDocumentFields = params.get(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS); + if (collapsedDocumentFields == null || "".equals(collapsedDocumentFields.trim())) { + return null; + } + + Set fields; + if ("*".equals(collapsedDocumentFields.trim())) { + fields = null; + } else { + fields = new HashSet(Arrays.asList(splitPattern.split(collapsedDocumentFields))); + } + + return new CollapsedDocumentCollapseCollector(request.getSearcher(), fields); + } + + + //============================================== Inner classes ======================================================= + + private static class CollapsedDocumentCollapseCollector implements CollapseCollector { + + private final Map collapsedDocumentsByFieldValue = new HashMap(); + private final Set includeCollapsedDocumentsFields; + private final SolrIndexSearcher searcher; + + public CollapsedDocumentCollapseCollector(SolrIndexSearcher searcher, Set fields) { + this.includeCollapsedDocumentsFields = fields; + this.searcher = searcher; + } + + public void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + OpenBitSet bitSet = collapsedDocumentsByFieldValue.get(collapseGroup); + if (bitSet == null) { + collapsedDocumentsByFieldValue.put(collapseGroup, bitSet = new OpenBitSet()); + } + // We cannot use fastset because we do not know the size upfront. + // The set method is more expensive than fastset, but that is the best for now. + // Setting bitSet to maxDoc may increase the memory footprint dramatically, because for each distinct fieldvalue + // there is a bitset + bitSet.set(docId); + } + + public void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + } + + @SuppressWarnings("unchecked") + public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { + IndexSchema schema = searcher.getSchema(); + DocumentBuilder documentBuilder = new DocumentBuilder(schema); + NamedList collapsedDocs = new NamedList(); + + Map docHeadCollapseGroupAssoc = collapseContext.getDocumentHeadCollapseGroupAssociation(); + + try { + for (DocIterator i = docs.iterator(); i.hasNext();) { + int id = i.nextDoc(); + if (!docHeadCollapseGroupAssoc.containsKey(id)) { + continue; + } + + CollapseGroup collapseGroup = docHeadCollapseGroupAssoc.get(id); + OpenBitSet bitSet = collapsedDocumentsByFieldValue.get(collapseGroup); + SolrDocumentList documentList = new SolrDocumentList(); + for (DocIdSetIterator iterator = bitSet.iterator(); iterator.nextDoc() != DocIdSetIterator.NO_MORE_DOCS;) { + Document luceneDocument = searcher.doc(iterator.docID(), includeCollapsedDocumentsFields); + + // the luceneDocument may still contain fields that are not specified in the includeCollapsedDocumentsFields + // this is because the document was read from the document cache. Off course we only want to return requested the fields. + // There is also another doc() method that accepts a FieldSelector instead of a list of fields, but that + // method does not use the Solr document cache, which from my perspective is a necessity + if (includeCollapsedDocumentsFields != null && luceneDocument.getFields().size() > includeCollapsedDocumentsFields.size()) { + Document document = new Document(); + for (Object object : luceneDocument.getFields()) { + Fieldable field = (Fieldable) object; + if (includeCollapsedDocumentsFields.contains(field.name())) { + document.add(field); + } + } + luceneDocument = document; + } + + SolrDocument solrDocument = new SolrDocument(); + documentBuilder.loadStoredFields(solrDocument, luceneDocument); + documentList.add(solrDocument); + } + documentList.setNumFound(bitSet.cardinality()); + collapsedDocs.add(collapseGroup.getKey(), documentList); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + + result.add("collapsedDocs", collapsedDocs); + } + } + +} Index: src/test/org/apache/solr/search/fieldcollapse/FieldCollapsingIntegrationTest.java =================================================================== --- src/test/org/apache/solr/search/fieldcollapse/FieldCollapsingIntegrationTest.java Sun Oct 25 23:11:29 CET 2009 +++ src/test/org/apache/solr/search/fieldcollapse/FieldCollapsingIntegrationTest.java Sun Oct 25 23:11:29 CET 2009 @@ -0,0 +1,476 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryResponse; +import org.apache.solr.search.DocSlice; +import org.apache.solr.update.AddUpdateCommand; +import org.apache.solr.update.DeleteUpdateCommand; +import org.apache.solr.util.AbstractSolrTestCase; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Integration tests for field collapsing functionality. + */ +public class FieldCollapsingIntegrationTest extends AbstractSolrTestCase { + + public String getSchemaFile() { + return "schema-fieldcollapse.xml"; + } + + public String getSolrConfigFile() { + return "solrconfig-fieldcollapse.xml"; + } + + public void testNonAdjacentFieldCollapse() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree"); + addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree"); + addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree"); + addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put("q", new String[]{"tree"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(2, fieldValueCounts.size()); + assertEquals(2, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(2, docIdCounts.size()); + assertEquals(2, docIdCounts.get("1")); + assertEquals(1, docIdCounts.get("4")); + } + + public void testNonAdjacentFieldCollapse_sortOnNameAndCollectAggregates() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "title5", "stock", "1", "price", "1.00"); + addToIndex("id", "2", "name", "author1", "title", "title4", "stock", "2", "price", "2.00"); + addToIndex("id", "3", "name", "author2", "title", "title3", "stock", "1", "price", "1.00"); + addToIndex("id", "4", "name", "author2", "title", "title2", "stock", "2", "price", "2.00"); + addToIndex("id", "5", "name", "author1", "title", "title1", "stock", "1", "price", "1.00"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CommonParams.Q, new String[]{"*:*"}); + parameters.put(CommonParams.SORT, new String[]{"title asc"}); + parameters.put(CollapseParams.COLLAPSE_AGGREGATE, new String[]{"sum(stock), avg(price), max(id), min(id)"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(2, fieldValueCounts.size()); + assertEquals(2, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(2, docIdCounts.size()); + assertEquals(2, docIdCounts.get("5")); + assertEquals(1, docIdCounts.get("4")); + + NamedList aggregatedResults = (NamedList) collapseCounts.get("aggregatedResults"); + NamedList stockAggregatedResult = aggregatedResults.get("sum(stock)"); + assertEquals("3", stockAggregatedResult.get("author1")); + assertEquals("1", stockAggregatedResult.get("author2")); + NamedList priceAggregatedResult = aggregatedResults.get("avg(price)"); + assertEquals("1.50", priceAggregatedResult.get("author1")); + assertEquals("1.00", priceAggregatedResult.get("author2")); + NamedList idMaxAggregatedResult = aggregatedResults.get("max(id)"); + assertEquals("2", idMaxAggregatedResult.get("author1")); + assertEquals("3", idMaxAggregatedResult.get("author2")); + NamedList idMinAggregatedResult = aggregatedResults.get("min(id)"); + assertEquals("1", idMinAggregatedResult.get("author1")); + assertEquals("3", idMinAggregatedResult.get("author2")); + } + + /* Also tests the dropout bug */ + public void testNonAdjacentFieldCollapse_sortOnNameAndCollectCollapsedDocs() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "title5"); + addToIndex("id", "2", "name", "author1", "title", "title4"); + addToIndex("id", "3", "name", "author2", "title", "title3"); + addToIndex("id", "4", "name", "author2", "title", "title2"); + addToIndex("id", "5", "name", "author1", "title", "title1"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CommonParams.Q, new String[]{"*:*"}); + parameters.put(CommonParams.SORT, new String[]{"title asc"}); + parameters.put(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, new String[]{"id"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(2, fieldValueCounts.size()); + assertEquals(2, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(2, docIdCounts.size()); + assertEquals(2, docIdCounts.get("5")); + assertEquals(1, docIdCounts.get("4")); + + NamedList collapsedDocs = (NamedList) collapseCounts.get("collapsedDocs"); + assertEquals(2, collapsedDocs.size()); + + assertEquals(2, collapsedDocs.get("author1").getNumFound()); + assertEquals(1, collapsedDocs.get("author1").get(0).getFieldValue("id")); + assertNull(collapsedDocs.get("author1").get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author1").get(0).getFieldValue("author")); + assertEquals(2, collapsedDocs.get("author1").get(1).getFieldValue("id")); + assertNull(collapsedDocs.get("author1").get(1).getFieldValue("title")); + assertNull(collapsedDocs.get("author1").get(1).getFieldValue("author")); + + assertEquals(1, collapsedDocs.get("author2").getNumFound()); + assertEquals(3, collapsedDocs.get("author2").get(0).getFieldValue("id")); + assertNull(collapsedDocs.get("author2").get(0).getFieldValue("author")); + assertNull(collapsedDocs.get("author2").get(0).getFieldValue("title")); + } + + public void testNonAdjacentCollapse_withFacetingBefore() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree"); + addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree"); + addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree"); + addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()}); + parameters.put("facet", new String[]{"true"}); + parameters.put("facet.field", new String[]{"name"}); + parameters.put("q", new String[]{"tree"}); + parameters.put("fq", new String[]{"*:*"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + SimpleOrderedMap facetCounts = (SimpleOrderedMap) response.getValues().get("facet_counts"); + SimpleOrderedMap facetFields = (SimpleOrderedMap) facetCounts.get("facet_fields"); + NamedList counts = (NamedList) facetFields.get("name"); + assertEquals(3, counts.get("author1")); + assertEquals(2, counts.get("author2")); + } + + public void testNonAdjacentFieldCollapse_withFacetingAfter() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree"); + addToIndex("id", "3", "name", "author2", "title", "a lot of words and a tree"); + addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree"); + addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the word tree"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.AFTER.toString()}); + parameters.put("facet", new String[]{"true"}); + parameters.put("facet.field", new String[]{"name"}); + parameters.put("q", new String[]{"tree"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + SimpleOrderedMap facetCounts = (SimpleOrderedMap) response.getValues().get("facet_counts"); + SimpleOrderedMap facetFields = (SimpleOrderedMap) facetCounts.get("facet_fields"); + NamedList counts = (NamedList) facetFields.get("name"); + assertEquals(1, counts.get("author1")); + assertEquals(1, counts.get("author2")); + } + + public void testAdjacentFieldCollapse() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree"); + addToIndex("id", "3", "name", "author2", "title", "tree tree and yet another tree and a random word"); + addToIndex("id", "4", "name", "author2", "title", "a lot of words and a tree"); + addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the words tree tree"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{"adjacent"}); + parameters.put(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, new String[]{"id,title"}); + parameters.put("q", new String[]{"tree"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(3, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(2, fieldValueCounts.size()); + assertEquals(1, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(2, docIdCounts.size()); + assertEquals(1, docIdCounts.get("1")); + assertEquals(1, docIdCounts.get("3")); + + NamedList collapsedDocs = (NamedList) collapseCounts.get("collapsedDocs"); + assertEquals(2, collapsedDocs.size()); + + assertEquals(1, collapsedDocs.get("author1").getNumFound()); + assertEquals(2, collapsedDocs.get("author1").get(0).getFieldValue("id")); + assertEquals("a tree with a tree", collapsedDocs.get("author1").get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author1").get(0).getFieldValue("author")); + + assertEquals(1, collapsedDocs.get("author2").getNumFound()); + assertEquals(4, collapsedDocs.get("author2").get(0).getFieldValue("id")); + assertEquals("a lot of words and a tree", collapsedDocs.get("author2").get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author2").get(0).getFieldValue("author")); + } + + public void testAdjacentFieldCollapse_includeCollapsedDocs() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree"); + addToIndex("id", "3", "name", "author2", "title", "tree tree and yet another tree and a random word"); + addToIndex("id", "4", "name", "author2", "title", "a lot of words and a tree"); + addToIndex("id", "5", "name", "author1", "title", "again a lot of garbage words and the words tree tree"); + addToIndex("id", "6", "name", "author1", "title", "again a lot of garbage words and the word tree"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{"adjacent"}); + parameters.put(CollapseParams.COLLAPSE_INCLUDE_COLLAPSED_DOCS_FIELDS, new String[]{"id,title"}); + parameters.put("q", new String[]{"tree"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(3, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(3, docIdCounts.size()); + assertEquals(1, docIdCounts.get("1")); + assertEquals(1, docIdCounts.get("3")); + assertEquals(1, docIdCounts.get("5")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(3, fieldValueCounts.size()); + assertEquals(1, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + assertEquals(1, fieldValueCounts.get("author1")); + + NamedList collapsedDocs = (NamedList) collapseCounts.get("collapsedDocs"); + assertEquals(3, collapsedDocs.size()); + + assertEquals(1, collapsedDocs.get("author1", 0).getNumFound()); + assertEquals(2, collapsedDocs.get("author1").get(0).getFieldValue("id")); + assertEquals("a tree with a tree", collapsedDocs.get("author1").get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author1").get(0).getFieldValue("author")); + + assertEquals(1, collapsedDocs.get("author2").getNumFound()); + assertEquals(4, collapsedDocs.get("author2").get(0).getFieldValue("id")); + assertEquals("a lot of words and a tree", collapsedDocs.get("author2").get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author2").get(0).getFieldValue("author")); + + assertEquals(1, collapsedDocs.get("author1", 1).getNumFound()); + assertEquals(6, collapsedDocs.get("author1", 1).get(0).getFieldValue("id")); + assertEquals("again a lot of garbage words and the word tree", collapsedDocs.get("author1", 1).get(0).getFieldValue("title")); + assertNull(collapsedDocs.get("author1", 1).get(0).getFieldValue("author")); + } + + public void testAdjacentFieldCollapse_includeAggregatedResults() throws Exception { + addToIndex("id", "1", "name", "author1", "title", "a tree", "stock", "1", "price", "1.00"); + addToIndex("id", "2", "name", "author1", "title", "a tree with a tree", "stock", "1", "price", "3.00"); + addToIndex("id", "3", "name", "author1", "title", "a tree with a tree and other tree", "stock", "1", "price", "1.00"); + addToIndex("id", "4", "name", "author2", "title", "tree tree and yet another tree and a random word", "stock", "1", "price", "1.00"); + addToIndex("id", "5", "name", "author2", "title", "a lot of words and a tree", "stock", "2", "price", "2.00"); + addToIndex("id", "6", "name", "author1", "title", "again a lot of garbage words and the words tree tree", "stock", "1", "price", "1.00"); + addToIndex("id", "7", "name", "author1", "title", "again a lot of garbage words and the word tree", "stock", "3", "price", "1.50"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{"adjacent"}); + parameters.put(CollapseParams.COLLAPSE_AGGREGATE, new String[]{"sum(stock), avg(price), min(id), max(id)"}); + parameters.put("q", new String[]{"tree"}); + request.setParams(new ModifiableSolrParams(parameters)); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(3, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(3, docIdCounts.size()); + assertEquals(2, docIdCounts.get("1")); + assertEquals(1, docIdCounts.get("4")); + assertEquals(1, docIdCounts.get("6")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(3, fieldValueCounts.size()); + assertEquals(2, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + assertEquals(1, fieldValueCounts.get("author1", 1)); + + NamedList aggregatedResults = (NamedList) collapseCounts.get("aggregatedResults"); + assertEquals(4, aggregatedResults.size()); + NamedList stockAggregatedResult = aggregatedResults.get("sum(stock)"); + assertEquals(3, stockAggregatedResult.size()); + assertEquals("2", stockAggregatedResult.get("author1")); + assertEquals("2", stockAggregatedResult.get("author2")); + assertEquals("3", stockAggregatedResult.get("author1", 1)); + NamedList priceAggregatedResult = aggregatedResults.get("avg(price)"); + assertEquals(3, priceAggregatedResult.size()); + assertEquals("2.00", priceAggregatedResult.get("author1")); + assertEquals("2.00", priceAggregatedResult.get("author2")); + assertEquals("1.50", priceAggregatedResult.get("author1", 1)); + NamedList minIdAggregatedResult = aggregatedResults.get("min(id)"); + assertEquals(3, minIdAggregatedResult.size()); + assertEquals("2", minIdAggregatedResult.get("author1")); + assertEquals("5", minIdAggregatedResult.get("author2")); + assertEquals("7", minIdAggregatedResult.get("author1", 1)); + NamedList maxIdAggregatedResult = aggregatedResults.get("max(id)"); + assertEquals(3, maxIdAggregatedResult.size()); + assertEquals("3", maxIdAggregatedResult.get("author1")); + assertEquals("5", maxIdAggregatedResult.get("author2")); + assertEquals("7", maxIdAggregatedResult.get("author1", 1)); + } + + public void testForArrayOutOfBoundsBugWhenSorting() throws Exception{ + addToIndex("id", "1", "name", "author1", "title", "title5"); + addToIndex("id", "2", "name", "author1", "title", "title4"); + addToIndex("id", "3", "name", "author2", "title", "title3"); + addToIndex("id", "4", "name", "author2", "title", "title2"); + addToIndex("id", "5", "name", "author1", "title", "title1"); + assertU(commit()); + + SolrQueryRequest request = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CommonParams.Q, new String[]{"*:*"}); + parameters.put(CommonParams.SORT, new String[]{"title asc"}); + request.setParams(new ModifiableSolrParams(parameters)); + + // removing doc with id 1 till 4 will make this test fail. When removing doc with id 5 the error does not occur + deleteFromIndex("1"); + assertU(commit()); + + SolrQueryResponse response = h.queryAndResponse(null, request); + DocSlice docSlice = (DocSlice) response.getValues().get("response"); + assertEquals(2, docSlice.size()); + + NamedList collapseCounts = (NamedList) response.getValues().get("collapse_counts"); + assertEquals("name_plain", collapseCounts.get("field")); + + NamedList fieldValueCounts = (NamedList) collapseCounts.get("count"); + assertEquals(2, fieldValueCounts.size()); + assertEquals(1, fieldValueCounts.get("author1")); + assertEquals(1, fieldValueCounts.get("author2")); + + NamedList docIdCounts = (NamedList) collapseCounts.get("doc"); + assertEquals(2, docIdCounts.size()); + assertEquals(1, docIdCounts.get("5")); + assertEquals(1, docIdCounts.get("4")); + } + + // ================================================= Helpers ======================================================= + + protected void addToIndex(SolrInputDocument document) throws IOException { + SolrQueryRequest request = lrf.makeRequest(); + SolrQueryResponse response = new SolrQueryResponse(); + AddUpdateCommand command = new AddUpdateCommand(); + command.solrDoc = document; + request.getCore().getUpdateProcessingChain(null).createProcessor(request, response).processAdd(command); + } + + protected void deleteFromIndex(String id) throws IOException { + SolrQueryRequest request = lrf.makeRequest(); + SolrQueryResponse response = new SolrQueryResponse(); + DeleteUpdateCommand deleteCommand = new DeleteUpdateCommand(); + deleteCommand.id = id; + deleteCommand.fromCommitted = true; + deleteCommand.fromPending = true; + request.getCore().getUpdateProcessingChain(null).createProcessor(request, response).processDelete(deleteCommand); + } + + protected void addToIndex(String... fields) throws IOException { + SolrInputDocument document = createSolrInputDocument(fields); + addToIndex(document); + } + + protected SolrInputDocument createSolrInputDocument(String... fields) { + if (fields.length % 2 != 0) { + throw new IllegalArgumentException("Supply both the name and values, field array length must be even"); + } + + SolrInputDocument document = new SolrInputDocument(); + for (int i = 0; i < fields.length; i++) { + document.addField(fields[i], fields[++i]); + } + + return document; + } +} Index: src/java/org/apache/solr/search/fieldcollapse/AdjacentDocumentCollapser.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/AdjacentDocumentCollapser.java Tue Oct 20 12:36:32 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/AdjacentDocumentCollapser.java Tue Oct 20 12:36:32 CEST 2009 @@ -0,0 +1,158 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.search.FieldCache; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollector; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; + +import java.io.IOException; +import java.util.List; + +/** + * Adjacent collapsing behaviour. Has the same behaviour as {@link NonAdjacentDocumentCollapser} but with one extra constraint. + * The collapsing happens in an adjacent manner. Meaning that only documents that are next to each other in a continuous + * row and that have the same field value might be collapsed. + */ +public class AdjacentDocumentCollapser extends AbstractDocumentCollapser { + + /** + * See {@link super#AbstractDocumentCollapser(org.apache.solr.handler.component.ResponseBuilder, List)} + */ + public AdjacentDocumentCollapser(ResponseBuilder rb, List collapseCollectorFactories) throws IOException { + super(rb, collapseCollectorFactories); + } + + + // ================================================= Helpers ========================================================= + + /** + * Applies adjacent collapsing on the specified uncollapsedDocSet. + * + * @param uncollapsedDocset The uncollapsed docset + * @param values The fieldvalues the collapse on + */ + protected void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values) { + int docCount = 0; // how many documents we have processed + int repeatCount = 0; // how many times we have seen the same value in a + int collapseCount = 0; // how many documents we have collapsed in this + int collapseId = -1; // the document we're collapsing into + String collapseValue = null; + + + long startTime = System.currentTimeMillis(); + for (DocIterator i = uncollapsedDocset.iterator(); i.hasNext();) { + int currentId = i.nextDoc(); + String currentValue = values.lookup[values.order[currentId]]; + + // Initializing + if (collapseValue == null) { + repeatCount = 0; + collapseCount = 0; + collapseId = currentId; + collapseValue = currentValue; + + // Collapse the document if the field value is the same and + // we have a run of at least collapseThreshold uncollapsedDocset. + } else if (collapseValue.equals(currentValue)) { + if (++repeatCount >= collapseThreshold) { + collapseCount++; + for (CollapseCollector collector : collectors) { + CollapseGroup valueToCollapse = new AdjacentCollapseGroup(collapseId, currentValue); + collector.documentCollapsed(currentId, valueToCollapse, collapseContext); + } + } else { + addDoc(currentId); + } + } else { + addDoc(collapseId); + if (collapseCount > 0) { + CollapseGroup valueToCollapse = new AdjacentCollapseGroup(collapseId, collapseValue); + for (CollapseCollector collector : collectors) { + collector.documentHead(collapseId, valueToCollapse, collapseContext); + } + } + + repeatCount = 0; + collapseCount = 0; + collapseId = currentId; + collapseValue = currentValue; + } + + // Stop after collapseMaxDocs documents + if (++docCount >= collapseMaxDocs) { + break; + } + } + + if (collapseId != -1) { + addDoc(collapseId); + } + + if (collapseCount > 0) { + CollapseGroup valueToCollapse = new AdjacentCollapseGroup(collapseId, collapseValue); + for (CollapseCollector collector : collectors) { + collector.documentHead(collapseId, valueToCollapse, collapseContext); + } + } + + timeCollapsing = System.currentTimeMillis() - startTime; + } + + + // ============================================ Inner Classes ======================================================== + + private static class AdjacentCollapseGroup implements CollapseGroup { + + private final int firstDocumentId; + private final String fieldValue; + + private AdjacentCollapseGroup(int firstDocumentId, String fieldValue) { + this.firstDocumentId = firstDocumentId; + this.fieldValue = fieldValue; + } + + public String getKey() { + return fieldValue; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AdjacentCollapseGroup that = (AdjacentCollapseGroup) o; + + if (firstDocumentId != that.firstDocumentId) return false; + if (fieldValue != null ? !fieldValue.equals(that.fieldValue) : that.fieldValue != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = firstDocumentId; + result = 31 * result + (fieldValue != null ? fieldValue.hashCode() : 0); + return result; + } + } + +} Index: src/test/test-files/solr/conf/schema-fieldcollapse.xml =================================================================== --- src/test/test-files/solr/conf/schema-fieldcollapse.xml Sun Oct 25 13:53:50 CET 2009 +++ src/test/test-files/solr/conf/schema-fieldcollapse.xml Sun Oct 25 13:53:50 CET 2009 @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + title + id + + + + Index: src/test/org/apache/solr/handler/component/CollapseComponentTest.java =================================================================== --- src/test/org/apache/solr/handler/component/CollapseComponentTest.java Fri Oct 09 13:40:41 CEST 2009 +++ src/test/org/apache/solr/handler/component/CollapseComponentTest.java Fri Oct 09 13:40:41 CEST 2009 @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.search.fieldcollapse.AdjacentDocumentCollapser; +import org.apache.solr.search.fieldcollapse.NonAdjacentDocumentCollapser; +import org.apache.solr.util.AbstractSolrTestCase; + +import java.util.HashMap; +import java.util.Map; + +/** + * Tests for {@link org.apache.solr.handler.component.CollapseComponent}. + */ +public class CollapseComponentTest extends AbstractSolrTestCase { + + public String getSchemaFile() { + return "schema-fieldcollapse.xml"; + } + + public String getSolrConfigFile() { + return "solrconfig-fieldcollapse.xml"; + } + + public void testResolveCollpaseRequest_AdjacentCollapserFacetsBefore() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.ADJACENT.toString()}); + responseBuilder.req.setParams(new ModifiableSolrParams(parameters)); + + CollapseComponent collapseComponent = new CollapseComponent(); + CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder); + assertEquals(CollapseParams.CollapseFacet.BEFORE, request.getCollapseFacet()); + assertEquals(AdjacentDocumentCollapser.class, request.getCollapser().getClass()); + } + + public void testResolveCollpaseRequest_NonAdjacentCollapserFacetsAfter() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.AFTER.toString()}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.NORMAL.toString()}); + responseBuilder.req.setParams(new ModifiableSolrParams(parameters)); + + CollapseComponent collapseComponent = new CollapseComponent(); + CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder); + assertEquals(CollapseParams.CollapseFacet.AFTER, request.getCollapseFacet()); + assertEquals(NonAdjacentDocumentCollapser.class, request.getCollapser().getClass()); + } + + public void testResolveCollpaseRequest_requestIsNull() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map parameters = new HashMap(); + parameters.put(CollapseParams.COLLAPSE_FACET, new String[]{CollapseParams.CollapseFacet.BEFORE.toString()}); + parameters.put(CollapseParams.COLLAPSE_TYPE, new String[]{CollapseParams.CollapseType.ADJACENT.toString()}); + responseBuilder.req.setParams(new ModifiableSolrParams(parameters)); + + CollapseComponent collapseComponent = new CollapseComponent(); + CollapseComponent.CollapseRequest request = collapseComponent.resolveCollapseRequest(responseBuilder); + assertNull(request); + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/AggregateCollapseCollectorFactory.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/AggregateCollapseCollectorFactory.java Sun Oct 25 16:08:31 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/AggregateCollapseCollectorFactory.java Sun Oct 25 16:08:31 CET 2009 @@ -0,0 +1,261 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.lucene.search.FieldCache; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.fieldcollapse.CollapseGroup; +import org.apache.solr.search.fieldcollapse.collector.aggregate.*; +import org.apache.solr.util.plugin.NamedListInitializedPlugin; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A collapse collector factory that creates collapse collectors that create aggregate statistics based on the collapsed + * documents. + * + *

+ * If the collapse.aggregate is set with a string that describes what data to collect then a new collapse + * collector is created that collects this data. + * + *

+ * The string should be formatted acording to the following format: + * [function name]([field name])[, ] ... + * Example: avg(price), max(stock), sum(stock) + * + *

+ * When this collapse collector factory is configured this feature is not enabled by default. + */ +public class AggregateCollapseCollectorFactory implements CollapseCollectorFactory, NamedListInitializedPlugin { + + private final static Pattern splitPattern = Pattern.compile(","); + private final static Pattern aggregatePattern = Pattern.compile("(.+)\\((.+)\\)"); + + private final Map> aggregateFunctions = new HashMap>(); + + @SuppressWarnings("unchecked") + public void init(NamedList args) { + NamedList functionClassNames = (NamedList) args.get("aggregateFunctions"); + if (functionClassNames == null) { + aggregateFunctions.put("avg", AverageFunction.class); + aggregateFunctions.put("sum", SumFunction.class); + aggregateFunctions.put("min", MinFunction.class); + aggregateFunctions.put("max", MaxFunction.class); + return; + } + + for (Map.Entry entry : functionClassNames) { + Class functionClass; + try { + functionClass = (Class) Class.forName(entry.getValue()); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + aggregateFunctions.put(entry.getKey(), functionClass); + } + } + + /** + * Returns an new collapse collector instance that creates aggregate statictics if the collapse.aggregate paramter + * is set. + * + * @param request The specified request + * @return an new collapse collector instance if the collapse.aggregate parameter is set properly + */ + public CollapseCollector createCollapseCollector(SolrQueryRequest request) { + String aggregateParameterValue = request.getParams().get(CollapseParams.COLLAPSE_AGGREGATE); + if (aggregateParameterValue == null) { + return null; + } + + Map aggregateFunctions = resolveAggregateFunctions(aggregateParameterValue); + return new AggregateCollapseCollector(aggregateFunctions, request.getSearcher()); + } + + /** + * Resolves on a per field basis the aggregate function. + *

    + *
  • Splits the functions. E.g. collapse.aggregate=avg(price),sum(stock) into avg(price) and sum(stock) + *
  • Splits the functionname and fieldname from each function in order to resolve and instantiates the aggregate function. + * E.g. functionname = avg, fieldname = price + *
  • Looks up the function class and creates an instance of it. + *
+ * + * @param aggregateParameterValue The collapse.aggregate parameter value in the solr request + * @return an aggragate function per field stored in a map. + */ + protected Map resolveAggregateFunctions(String aggregateParameterValue) { + Map fieldNameAggregateFunction = new HashMap(); + + String[] aggregates = splitPattern.split(aggregateParameterValue); + for (String aggregate : aggregates) { + Matcher matcher = aggregatePattern.matcher(aggregate); + if (!matcher.matches() || matcher.groupCount() != 2) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Aggregate function syntax is incorrect."); + } + + String functionName = matcher.group(1).trim(); + String fieldName = matcher.group(2); + AggregateFunction function = resolveAggregateFunction(functionName); + fieldNameAggregateFunction.put(new AggregateField(fieldName, functionName), function); + } + + return fieldNameAggregateFunction; + } + + /** + * Returns a new instance of the AggregateFunction with the specified functionName. + * + * @param functionName The name of the function to create a new instance of + * @return a new instance of the AggregateFunction with the specified functionName + * @throws UnsupportedOperationException When no aggregate functions with the specified function name exists + */ + protected AggregateFunction resolveAggregateFunction(String functionName) { + Class classOfFunction = aggregateFunctions.get(functionName); + if (classOfFunction == null) { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, String.format("Aggregate function with name [%s] does not exist", functionName)); + } + + try { + return classOfFunction.newInstance(); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + + //============================================== Inner classes ======================================================= + + private static class AggregateCollapseCollector implements CollapseCollector { + + private final Map functions; + private final Map fieldCaches = new HashMap(); + private final Map fieldTypes = new HashMap(); + + private AggregateCollapseCollector(Map functions, SolrIndexSearcher searcher) { + this.functions = functions; + + for (AggregateField aggregateField : functions.keySet()) { + try { + String fieldName = aggregateField.getFieldName(); + fieldCaches.put(fieldName, FieldCache.DEFAULT.getStringIndex(searcher.getReader(), fieldName)); + fieldTypes.put(fieldName, searcher.getSchema().getFieldType(fieldName)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + public void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + for (AggregateField aggregateField : functions.keySet()) { + FieldCache.StringIndex stringIndex = fieldCaches.get(aggregateField.getFieldName()); + AggregateFunction function = functions.get(aggregateField); + + String fieldCacheValue = stringIndex.lookup[stringIndex.order[docId]]; + String value = fieldTypes.get(aggregateField.getFieldName()).indexedToReadable(fieldCacheValue); + function.collect(collapseGroup, value); + } + } + + public void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + } + + public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { + NamedList aggregateResults = new NamedList(); + Map docHeadCollapseGroupAssoc = collapseContext.getDocumentHeadCollapseGroupAssociation(); + + for (AggregateField aggregateField : functions.keySet()) { + NamedList resultFieldName = new NamedList(); + for (DocIterator i = docs.iterator(); i.hasNext();) { + int id = i.nextDoc(); + CollapseGroup collapseGroup = docHeadCollapseGroupAssoc.get(id); + if (collapseGroup == null) { + continue; + } + + AggregateFunction function = functions.get(aggregateField); + String functionResult = function.calculate(collapseGroup); + if (functionResult != null) { + resultFieldName.add(collapseGroup.getKey(), functionResult); + } + } + aggregateResults.add(aggregateField.getUniqueName(), resultFieldName); + } + + result.add("aggregatedResults", aggregateResults); + } + + } + + private final static class AggregateField { + + private final String fieldName; + private final String functionName; + + private AggregateField(String fieldName, String functionName) { + this.fieldName = fieldName; + this.functionName = functionName; + } + + public String getFieldName() { + return fieldName; + } + + public String getFunctionName() { + return functionName; + } + + public String getUniqueName() { + return String.format("%s(%s)", functionName, fieldName); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + AggregateField that = (AggregateField) o; + + if (fieldName != null ? !fieldName.equals(that.fieldName) : that.fieldName != null) return false; + if (functionName != null ? !functionName.equals(that.functionName) : that.functionName != null) return false; + + return true; + } + + @Override + public int hashCode() { + int result = fieldName != null ? fieldName.hashCode() : 0; + result = 31 * result + (functionName != null ? functionName.hashCode() : 0); + return result; + } + } + +} Index: src/java/org/apache/solr/search/DocSetHitCollector.java =================================================================== --- src/java/org/apache/solr/search/DocSetHitCollector.java (revision 794328) +++ src/java/org/apache/solr/search/DocSetHitCollector.java Fri Oct 09 13:40:42 CEST 2009 @@ -17,11 +17,10 @@ package org.apache.solr.search; -import org.apache.lucene.search.HitCollector; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.Collector; import org.apache.lucene.search.Scorer; import org.apache.lucene.util.OpenBitSet; -import org.apache.lucene.index.IndexReader; import java.io.IOException; @@ -29,7 +28,7 @@ * @version $Id$ */ -class DocSetCollector extends Collector { +class DocSetCollector extends DocSetAwareCollector { int pos=0; OpenBitSet bits; final int maxDoc; Index: src/java/org/apache/solr/search/fieldcollapse/AbstractDocumentCollapser.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/AbstractDocumentCollapser.java Sun Oct 25 20:38:36 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/AbstractDocumentCollapser.java Sun Oct 25 20:38:36 CET 2009 @@ -0,0 +1,370 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.util.OpenBitSet; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.search.*; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollector; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; +import org.apache.solr.search.fieldcollapse.collector.CollapseContext; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Base Collapser implementation + */ +public abstract class AbstractDocumentCollapser implements DocumentCollapser { + + /** + * Field to use to collapse results. Parameter. + */ + protected final String collapseField; + + /** + * Number of documents with the same value for collapseField after which + * collapsing kicks in. Parameter. + */ + protected final int collapseThreshold; + + /** Collapse State *********************************************** */ + + /** + * Maximum number of documents to process during field collapsing. + * Parameter. + */ + protected final int collapseMaxDocs; + + /** + * Whether to include debug information in the request + */ + protected boolean includeDebugInformation; + + /** + * Maxmimum size for a HashDocSet. + */ + protected final int hashMaxSize; + + /** + * Buffer for collecting documents. Gets turned into different types of + * DocSet depending on the number of documents we end up with. + */ + protected int[] docbuf; + + /** + * Number of documents in docbuf. + */ + protected int docbufSize = 0; + + /** + * Maximum document id currently in docbuf. Only valid while docbufSize < + * hashMaxSize. + */ + protected int docbufMaxDoc = 0; + + /** + * Bitset representation of docbuf. Gets created when docbufSize >= + * hashMaxSize. + */ + protected OpenBitSet docbufBitSet; + + /** + * The result of executing the query and filter queries, without collapsing. + */ + protected DocSet uncollapsedDocSet; + + protected FieldCache.StringIndex fieldValues; + protected final SolrIndexSearcher searcher; + protected final List collectors; + + + protected final int flags; + + protected final CollapseContext collapseContext; + + private final List collapseCollectorFactories; + + /** + * Debug Information ******************************************** + */ + protected long timeCollapsing = 0; + protected long timeConvertToBitSet = 0; + protected long timeCreateDocSet = 0; + protected long timeCreateCollapseInfo = 0; + protected long timeCreateUncollapedDocset = 0; + protected String debugDocSetInfo = "unknown"; + + + // ============================================ Constructors ======================================================= + + /** + * Constructs a base collapser. + * + * @param rb The response builder + * @param collapseCollectorFactories A list of collapse collector factories configured for the document collapser + * @throws IOException if index searcher related problems occur + */ + protected AbstractDocumentCollapser(ResponseBuilder rb, List collapseCollectorFactories) throws IOException { + this.collapseCollectorFactories = collapseCollectorFactories; + + // Allocate data structures + hashMaxSize = rb.req.getCore().getSolrConfig().hashDocSetMaxSize; + docbuf = new int[hashMaxSize]; + this.searcher = rb.req.getSearcher(); + this.flags = rb.getFieldFlags(); + + // parsing Solr parameters + SolrParams params = rb.req.getParams(); + collapseField = params.required().get(CollapseParams.COLLAPSE_FIELD); + Integer ct = params.getInt(CollapseParams.COLLAPSE_THRESHOLD); + if (ct == null) { + ct = params.getInt(CollapseParams.COLLAPSE_MAX); + } + collapseThreshold = (ct != null) ? ct : 1; + checkCollapseField(rb.req.getSchema()); + + int collapseMaxDocs = params.getInt(CollapseParams.COLLAPSE_MAXDOCS, 0); + this.collapseMaxDocs = collapseMaxDocs <= 0 ? searcher.maxDoc() : collapseMaxDocs; + includeDebugInformation = params.getBool(CollapseParams.COLLAPSE_DEBUG ,false); + + collapseContext = new CollapseContext(); + collectors = initializeCollapseCollectors(rb.req); + } + + + // ======================================= Public API methods ======================================================== + + /** + * {@inheritDoc} + */ + public DocumentCollapseResult collapse(Query mainQuery, List filterQueries, Sort sort) throws IOException { + long startTime = System.currentTimeMillis(); + doQuery(mainQuery, filterQueries, sort); + timeCreateUncollapedDocset = System.currentTimeMillis() - startTime; + fieldValues = FieldCache.DEFAULT.getStringIndex(searcher.getReader(), collapseField); + doCollapsing(uncollapsedDocSet, fieldValues); + return createDocumentCollapseResult(); + } + + /** + * {@inheritDoc} + */ + public NamedList getCollapseInfo(SolrIndexSearcher searcher, DocList docs) throws IOException { + if (uncollapsedDocSet == null) { + throw new IllegalStateException("Invoke the collapse method before invoking getCollapseInfo method"); + } + + long startTime = System.currentTimeMillis(); + + NamedList result = new NamedList(); + result.add("field", collapseField); + + for (CollapseCollector collapseCollector : collectors) { + collapseCollector.getResult(result, docs, collapseContext); + } + + if (includeDebugInformation) { + result.add("debug", getDebugInfo()); + } + + timeCreateCollapseInfo = System.currentTimeMillis() - startTime; + return result; + } + + + // ================================================= Helpers ======================================================= + + /** + * Executes implementation specific collapsing algorithm. + * + * @param uncollapsedDocset The uncollapsed DocSet + * @param values The field values + */ + protected abstract void doCollapsing(DocSet uncollapsedDocset, FieldCache.StringIndex values); + + /** + * @return timing information for field collapsing process. + */ + protected NamedList getDebugInfo() { + long totalTime = timeCreateUncollapedDocset + timeCollapsing + timeCreateCollapseInfo + timeConvertToBitSet + timeCreateDocSet; + NamedList namedList = new NamedList(); + namedList.add("Docset type", debugDocSetInfo); + namedList.add("Total collapsing time(ms)", totalTime); + namedList.add("Create uncollapsed docset(ms)", timeCreateUncollapedDocset); + namedList.add(String.format("%s collapsing time(ms)", getClass().getSimpleName()), timeCollapsing); + namedList.add("Creating collapseinfo time(ms)", timeCreateCollapseInfo); + namedList.add("Convert to bitset time(ms)", timeConvertToBitSet); + namedList.add("Create collapsed docset time(ms)", timeCreateDocSet); + return namedList; + } + + /** + * Creates the document collapse result based on the uncollapsed docset and the collapsed docset. + * + * @return he document collapse result + */ + protected DocumentCollapseResult createDocumentCollapseResult() { + long startTime = System.currentTimeMillis(); + DocSet result = (docbufBitSet != null) ? new BitDocSet(docbufBitSet) : new HashDocSet(docbuf, 0, docbufSize); + timeCreateDocSet = System.currentTimeMillis() - startTime; + debugDocSetInfo = result.getClass().getSimpleName() + "(" + docbufSize + ")"; + return new DocumentCollapseResult(result, uncollapsedDocSet); + } + + /** + * Adds a document to the internal document buffer. + * + * @param doc The lucene identifier of the document to add + */ + protected void addDoc(int doc) { + // If we have less than hashMaxSize documents, just + // keep adding them to docbuf. We will turn them into + // a HashDocSet later. + if (docbufSize < hashMaxSize) { + docbuf[docbufSize] = doc; + if (doc > docbufMaxDoc) { + docbufMaxDoc = doc; + } + } else { + // We have exceeded hashMaxSize. Allocate a bit set + // if we don't have one yet, then add to that. + if (docbufBitSet == null) { + long startTime = System.currentTimeMillis(); + docbufBitSet = new OpenBitSet(docbufMaxDoc + 1); + for (int i = 0; i < docbufSize; i++) { + docbufBitSet.fastSet(docbuf[i]); + } + timeConvertToBitSet = System.currentTimeMillis() - startTime; + } + docbufBitSet.set(doc); + } + docbufSize++; + } + + protected void doQuery(Query mainQuery, List filterQueries, Sort sort) throws IOException { + uncollapsedDocSet = searcher.getDocList(mainQuery, filterQueries, sort, 0, collapseMaxDocs, flags); + } + + /** + * Method that checks if proper field collapsing is actually possible with the current collapse field. + * If the collapse field does not meet the proper requirements a runtime exception is thrown. + * A runtime exception is thrown under the following circumstances: + *
    + *
  • When the collapse field does not exists in the schema + *
  • When the collapse field is multivalued in the schema + *
  • When the collapse field is tokenized in the schema + *
+ * + * For example when a field is tokenized, only the last token of the field can be retrieved from the fieldcache. This + * results in field collapsing only on the last token of a field value instead of the complete field value. + * + * When the field values from a multivalued field are returned from the FieldCache then an exception may + * be thrown. This happens when there are more terms in a field than documents. + * + * @param schema The index schema + */ + protected void checkCollapseField(IndexSchema schema) { + SchemaField schemaField = schema.getFieldOrNull(collapseField); + if (schemaField == null) { + throw new RuntimeException("Could not collapse, because collapse field does not exist in the schema."); + } + + if (schemaField.multiValued()) { + throw new RuntimeException("Could not collapse, because collapse field is multivalued"); + } + + if (schemaField.getType().isTokenized()) { + throw new RuntimeException("Could not collapse, because collapse field is tokenized"); + } + } + + protected List initializeCollapseCollectors(SolrQueryRequest request) { + List collectors = new ArrayList(); + for (CollapseCollectorFactory factory : collapseCollectorFactories) { + CollapseCollector collapseCollector = factory.createCollapseCollector(request); + if (collapseCollector != null) { + if (collectors.isEmpty()) { + collectors.add(new DocumentHeadCollapseGroupCollector()); + } + + collectors.add(collapseCollector); + } + } + return collectors; + } + + // =========================================== Getters / Setters =================================================== + + /** + * @return field to use to collapse results + */ + public String getCollapseField() { + return collapseField; + } + + /** + * @return maximum number of documents to process during field collapsing + */ + public int getCollapseMaxDocs() { + return collapseMaxDocs; + } + + /** + * @return number of documents with the same value for collapseField after which collapsing kicks in. + */ + public int getCollapseThreshold() { + return collapseThreshold; + } + + + //============================================== Inner classes ======================================================= + + // Collector that associates a collapse group with the document head identifier. + // This is information that many collapse collectors utilize. + // This collector has to be the first collector. Order does matter. + public static class DocumentHeadCollapseGroupCollector implements CollapseCollector { + + private final Map docHeadIdCollapseGroupAssoc = new HashMap(); + + public void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + } + + public void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + docHeadIdCollapseGroupAssoc.put(docHeadId, collapseGroup); + if (collapseContext.getDocumentHeadCollapseGroupAssociation() == null) { + collapseContext.setDocumentHeadCollapseGroupAssociation(docHeadIdCollapseGroupAssoc); + } + } + + public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { + } + } +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AggregateFunction.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AggregateFunction.java Sun Oct 18 23:05:38 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/aggregate/AggregateFunction.java Sun Oct 18 23:05:38 CEST 2009 @@ -0,0 +1,52 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector.aggregate; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +/** + * A concrete aggregate function implementation is responsible for calculating the aggregated information and is used + * by {@link org.apache.solr.search.fieldcollapse.collector.AggregateCollapseCollectorFactory} + * and its CollapseCollector to provide the aggregated data. + */ +public interface AggregateFunction { + + /** + * Adds the specified number that a list of other numbers that are associated with the specified field (group) + * + * @param collapseGroup The field value (group) to add the number to. + * @param number The number to add + */ + void collect(CollapseGroup collapseGroup, String number); + + /** + * Returns the string representation of the calculated aggregate for the specified collapseGroup. + * + * @param collapseGroup The collapse group to calculate the aggragate data for. + * @return the calculated aggregates per field value (group) + */ + String calculate(CollapseGroup collapseGroup); + + /** + * Returns the name of the function. + * + * @return the name of the function + */ + String getName(); + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/FieldValueCountCollapseCollectorFactory.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/FieldValueCountCollapseCollectorFactory.java Sun Oct 25 22:37:53 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/FieldValueCountCollapseCollectorFactory.java Sun Oct 25 22:37:53 CET 2009 @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocList; +import org.apache.solr.search.SolrIndexSearcher; +import org.apache.solr.search.fieldcollapse.CollapseGroup; +import org.apache.solr.search.fieldcollapse.util.Counter; + +import java.util.HashMap; +import java.util.Map; + +/** + * A collapse collector factory that creates collapse collectors that collect t the collapse count per collapsed group + * and return the counts in the response per collepsed group field value. When this collapse collector factory is + * configured it is enabled by default. To disable this feature for a certain search add collapse.info.count=false + * to the request as parameter. + */ +public class FieldValueCountCollapseCollectorFactory implements CollapseCollectorFactory { + + /** + * {@inheritDoc} + */ + public CollapseCollector createCollapseCollector(SolrQueryRequest request) { + SolrParams params = request.getParams(); + String collapseField = params.required().get(CollapseParams.COLLAPSE_FIELD); + boolean includeFieldValueCount = params.getBool(CollapseParams.COLLAPSE_INFO_COUNT, true); + return includeFieldValueCount ? new FieldValueCountCollapseCollector(request.getSearcher(), collapseField) : null; + } + + + //============================================== Inner classes ======================================================= + + private static class FieldValueCountCollapseCollector implements CollapseCollector { + + private final FieldType collapseFieldType; + private final Map fieldValueCount = new HashMap(); + + public FieldValueCountCollapseCollector(SolrIndexSearcher searcher, String collapseField) { + IndexSchema schema = searcher.getSchema(); + collapseFieldType = schema.getField(collapseField).getType(); + } + + public void documentCollapsed(int docId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + Counter counter = fieldValueCount.get(collapseGroup); + if (counter == null) { + counter = new Counter(0); + fieldValueCount.put(collapseGroup, counter); + } + counter.increment(); + } + + public void documentHead(int docHeadId, CollapseGroup collapseGroup, CollapseContext collapseContext) { + } + + public void getResult(NamedList result, DocList docs, CollapseContext collapseContext) { + NamedList resCount = new NamedList(); + Map docHeadCollapseGroupAssoc = collapseContext.getDocumentHeadCollapseGroupAssociation(); + + for (DocIterator i = docs.iterator(); i.hasNext();) { + int id = i.nextDoc(); + CollapseGroup collapseGroup = docHeadCollapseGroupAssoc.get(id); + if (collapseGroup == null) { + continue; + } + + Integer count = fieldValueCount.get(collapseGroup).getCount(); + resCount.add(collapseFieldType.indexedToReadable(collapseGroup.getKey()), count); + } + + result.add("count", resCount); + } + + } +} Index: src/test/org/apache/solr/search/fieldcollapse/AdjacentCollapserTest.java =================================================================== --- src/test/org/apache/solr/search/fieldcollapse/AdjacentCollapserTest.java Sun Oct 25 19:32:13 CET 2009 +++ src/test/org/apache/solr/search/fieldcollapse/AdjacentCollapserTest.java Sun Oct 25 19:32:13 CET 2009 @@ -0,0 +1,180 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse; + +import org.apache.lucene.search.FieldCache; +import org.apache.solr.common.params.CollapseParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.handler.component.ResponseBuilder; +import org.apache.solr.search.DocIterator; +import org.apache.solr.search.DocSet; +import org.apache.solr.search.HashDocSet; +import org.apache.solr.search.fieldcollapse.collector.CollapseCollectorFactory; +import org.apache.solr.search.fieldcollapse.collector.DocumentGroupCountCollapseCollectorFactory; +import org.apache.solr.search.fieldcollapse.util.Counter; +import org.apache.solr.util.AbstractSolrTestCase; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for {@link AdjacentDocumentCollapser}. + */ +public class AdjacentCollapserTest extends AbstractSolrTestCase { + + private AdjacentDocumentCollapser adjacentCollapser; + + public String getSchemaFile() { + return "schema-fieldcollapse.xml"; + } + + public String getSolrConfigFile() { + return "solrconfig-fieldcollapse.xml"; + } + + /* + * Pass condition: document with id 2 and 6 are collapsed + */ + public void testAdjacentCollapse() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map params = new HashMap(); + params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"}); + responseBuilder.req.setParams(new ModifiableSolrParams(params)); + + + CollapseCollectorFactory factory = new DocumentGroupCountCollapseCollectorFactory(); + adjacentCollapser = new AdjacentDocumentCollapser(responseBuilder, Arrays.asList(factory)); + // adjacent collapsing can only works with an order docset + String[] values = new String[]{"a", "b", "c", "z"}; + int[] order = new int[]{3, 0, 0, 1, 2, 1, 1}; + FieldCache.StringIndex index = new FieldCache.StringIndex(order, values); + int[] docIds = new int[]{0, 1, 2, 3, 4, 5, 6}; + DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7); + adjacentCollapser.doCollapsing(uncollapsedDocset, index); + + Map collapseCounts = (Map) adjacentCollapser.collapseContext.get("documentHeadCount"); + assertEquals(2, collapseCounts.size()); + assertEquals(1, collapseCounts.get(1).getCount()); + assertEquals(1, collapseCounts.get(5).getCount()); + + DocSet docHeadsDocSet = adjacentCollapser.createDocumentCollapseResult().getCollapsedDocset(); + assertEquals(5, docHeadsDocSet.size()); + + boolean zeroFound = false; + boolean oneFound = false; + boolean twoFound = false; + boolean threeFound = false; + boolean fourFound = false; + boolean fiveFound = false; + boolean sixFound = false; + for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) { + int docId = i.nextDoc(); + if (docId == 0) { + zeroFound = true; + } else if (docId == 1) { + oneFound = true; + } else if (docId == 2) { + twoFound = true; + } else if (docId == 3) { + threeFound = true; + } else if (docId == 4) { + fourFound = true; + } else if (docId == 5) { + fiveFound = true; + } else if (docId == 6) { + sixFound = true; + } + } + + assertTrue(zeroFound); + assertTrue(oneFound); + assertFalse(twoFound); + assertTrue(threeFound); + assertTrue(fourFound); + assertTrue(fiveFound); + assertFalse(sixFound); + } + + /* + * Pass condition: only document with id 6 is collapsed + */ + public void testAdjacentCollapse_CollapseThresholdTwo() throws Exception { + ResponseBuilder responseBuilder = new ResponseBuilder(); + responseBuilder.req = lrf.makeRequest(); + Map params = new HashMap(); + params.put(CollapseParams.COLLAPSE_FIELD, new String[]{"name_plain"}); + params.put(CollapseParams.COLLAPSE_MAXDOCS, new String[]{"10"}); + params.put(CollapseParams.COLLAPSE_THRESHOLD, new String[]{"2"}); + responseBuilder.req.setParams(new ModifiableSolrParams(params)); + + CollapseCollectorFactory factory = new DocumentGroupCountCollapseCollectorFactory(); + adjacentCollapser = new AdjacentDocumentCollapser(responseBuilder, Arrays.asList(factory)); + // adjacent collapsing can only works with an order docset + String[] values = new String[]{"a", "b", "c", "z"}; + int[] order = new int[]{3, 0, 0, 2, 1, 1, 1}; + FieldCache.StringIndex index = new FieldCache.StringIndex(order, values); + int[] docIds = new int[]{0, 1, 2, 3, 4, 5, 6}; + DocSet uncollapsedDocset = new HashDocSet(docIds, 0, 7); + adjacentCollapser.doCollapsing(uncollapsedDocset, index); + + Map collapseCounts = (Map) adjacentCollapser.collapseContext.get("documentHeadCount"); + assertEquals(1, collapseCounts.size()); + assertEquals(1, collapseCounts.get(4).getCount()); + + DocSet docHeadsDocSet = adjacentCollapser.createDocumentCollapseResult().getCollapsedDocset(); + assertEquals(6, docHeadsDocSet.size()); + + boolean zeroFound = false; + boolean oneFound = false; + boolean twoFound = false; + boolean threeFound = false; + boolean fourFound = false; + boolean fiveFound = false; + boolean sixFound = false; + for (DocIterator i = docHeadsDocSet.iterator(); i.hasNext();) { + int docId = i.nextDoc(); + if (docId == 0) { + zeroFound = true; + } else if (docId == 1) { + oneFound = true; + } else if (docId == 2) { + twoFound = true; + } else if (docId == 3) { + threeFound = true; + } else if (docId == 4) { + fourFound = true; + } else if (docId == 5) { + fiveFound = true; + } else if (docId == 6) { + sixFound = true; + } + } + + assertTrue(zeroFound); + assertTrue(oneFound); + assertTrue(twoFound); + assertTrue(threeFound); + assertTrue(fourFound); + assertTrue(fiveFound); + assertFalse(sixFound); + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/collector/CollapseContext.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/collector/CollapseContext.java Sun Oct 25 20:03:59 CET 2009 +++ src/java/org/apache/solr/search/fieldcollapse/collector/CollapseContext.java Sun Oct 25 20:03:59 CET 2009 @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.search.fieldcollapse.collector; + +import org.apache.solr.search.fieldcollapse.CollapseGroup; + +import java.util.HashMap; +import java.util.Map; + +/** + * A collapse context allows CollapseCollectors to share data between each other. + * Per search request only one CollapseContext will be created. + * A CollapseContext is not thread save. + */ +public class CollapseContext { + + private final Map data = new HashMap(); + + /** + * Sets a value in the collapse context under the specified key. + * + * @param key The key + * @param value The value to share + */ + public void put(String key, Object value) { + data.put(key, value); + } + + /** + * Returns a value in the collapse context that is associated with the specified key or null if + * no value is associated with the specified key. + * + * @param key The key + * @return a value in the collapse or null + */ + public Object get(String key) { + return data.get(key); + } + + /** + * Sets the documenthead collapse group association. + * + * @param documentHeadCollapseGroupAssociation The documenthead collapse group association + */ + public void setDocumentHeadCollapseGroupAssociation(Map documentHeadCollapseGroupAssociation) { + data.put("documentHeadCollapseGroupAssociation", documentHeadCollapseGroupAssociation); + } + + /** + * Returns the documenthead collapse group association. + * The documenthead collapse group association is a map with the document head as key and the collapse group as value. + * + * + * @return the documenthead collapse group association + */ + public Map getDocumentHeadCollapseGroupAssociation() { + return (Map) data.get("documentHeadCollapseGroupAssociation"); + } + +} Index: src/java/org/apache/solr/search/fieldcollapse/DocumentCollapseResult.java =================================================================== --- src/java/org/apache/solr/search/fieldcollapse/DocumentCollapseResult.java Fri Oct 09 13:40:41 CEST 2009 +++ src/java/org/apache/solr/search/fieldcollapse/DocumentCollapseResult.java Fri Oct 09 13:40:41 CEST 2009 @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search.fieldcollapse; + +import org.apache.solr.search.DocSet; + +/** + * Contains the collapsed and uncollapsed result. + */ +public class DocumentCollapseResult { + + private final DocSet collapsedDocset; + private final DocSet unCollapsedDocset; + + /** + * Constructs a DocumentCollapseResult. + * + * @param collapsedDocset The collapsed docset + * @param unCollapsedDocset The uncollapsed docset + */ + public DocumentCollapseResult(DocSet collapsedDocset, DocSet unCollapsedDocset) { + this.collapsedDocset = collapsedDocset; + this.unCollapsedDocset = unCollapsedDocset; + } + + /** + * Returns the collapsed docset. + * + * @return the collapsed docset + */ + public DocSet getCollapsedDocset() { + return collapsedDocset; + } + + /** + * Returns the uncollapsed docset. + * + * @return the uncollapsed docset + */ + public DocSet getUnCollapsedDocset() { + return unCollapsedDocset; + } +} Index: src/java/org/apache/solr/search/DocSetAwareCollector.java =================================================================== --- src/java/org/apache/solr/search/DocSetAwareCollector.java Fri Oct 09 13:40:41 CEST 2009 +++ src/java/org/apache/solr/search/DocSetAwareCollector.java Fri Oct 09 13:40:41 CEST 2009 @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.lucene.search.Collector; + +/** + * A collecter that is DocSet aware. + * When the search has been performed, the collected documents can be retrieved in the form of a DocSet. + */ +public abstract class DocSetAwareCollector extends Collector { + + /** + * Returns the collected documents in a DocSet. + * + * @return the collected documents + */ + public abstract DocSet getDocSet(); + +}