Property changes on: . ___________________________________________________________________ Name: svn:ignore - build dist .project .classpath .settings + build dist .project .classpath .settings field_collapsing_1.1.0.patch Index: src/java/org/apache/solr/search/CollapseFilter.java =================================================================== --- src/java/org/apache/solr/search/CollapseFilter.java (revision 0) +++ src/java/org/apache/solr/search/CollapseFilter.java (revision 0) @@ -0,0 +1,271 @@ +package org.apache.solr.search; + +import static org.apache.solr.request.SolrParams.COLLAPSE_FIELD; +import static org.apache.solr.request.SolrParams.COLLAPSE_MAX; +import static org.apache.solr.request.SolrParams.COLLAPSE_TYPE; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; + +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.solr.request.SolrParams; +import org.apache.solr.util.NamedList; + +/** + * + * @author Emmanuel Keller keller.emmanuel@gmail.com + */ +public class CollapseFilter { + + private enum CollapseType { + normal(0), adjacent(1); + + protected int value; + + CollapseType(int v) { + value = v; + } + } + + private class CollapseParams { + /** + * Field use to collapse results + */ + private String collapseField; + + /** + * Maximum number of visible documents for a collapsed result + */ + private int collapseMax; + + private CollapseType collapseType; + + private CollapseParams(SolrParams p) { + collapseField = p.get(COLLAPSE_FIELD); + collapseMax = p.getInt(COLLAPSE_MAX, 1); + if ("adjacent".equals(p.get(COLLAPSE_TYPE, "normal"))) + collapseType = CollapseType.adjacent; + else + collapseType = CollapseType.normal; + } + + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof CollapseParams)) + return false; + CollapseParams other = (CollapseParams) o; + if (collapseType != other.collapseType) + return false; + if (collapseMax != other.collapseMax) + return false; + if (!collapseField.equals(other.collapseField)) + return false; + return true; + } + } + + public class CollapseCacheKey { + + private QueryResultKey queryKey; + + private CollapseParams collapseParams; + + private final int hc; + + public CollapseCacheKey(QueryResultKey queryKey, CollapseParams collapseParams) { + this.queryKey = queryKey; + this.collapseParams = collapseParams; + hc = queryKey.hashCode(); + } + + public int hashCode() { + return hc; + } + + public boolean equals(Object o) { + if (o == this) + return true; + if (!(o instanceof CollapseCacheKey)) + return false; + CollapseCacheKey other = (CollapseCacheKey) o; + if (!collapseParams.equals(other.collapseParams)) + return false; + if (!queryKey.equals(other.queryKey)) + return false; + return true; + } + } + + private class CollapseResult { + /** + * Contains final documents (DocSet) + */ + private BitDocSet docSet; + + /** + * Subset of document having more results + */ + private BitDocSet hasMoreResult; + + /** + * Group list + */ + private HashMap collapseList; + + /** + * Stores the reader version. Used to invalidate old cached results + */ + private long readerVersion; + + private CollapseResult(long readerVersion) { + hasMoreResult = new BitDocSet(); + collapseList = new HashMap(); + docSet = new BitDocSet(); + this.readerVersion = readerVersion; + } + } + + /** + * Analyse of SolrParams + */ + private CollapseParams collapseParams = null; + + /** + * Results of collapsing (for cache storage) + */ + private CollapseResult collapseResult = null; + + /** + * + * @param searcher + * @param query + * @param filters + * @param sort + * @param flags + * @param params + * @throws IOException + */ + public CollapseFilter(SolrIndexSearcher searcher, Query query, List filters, Sort sort, int flags, + SolrParams params) throws IOException { + this.collapseParams = new CollapseParams(params); + + // Cache + CollapseCacheKey cacheKey = new CollapseCacheKey(new QueryResultKey(query, filters, sort, flags), collapseParams); + collapseResult = (CollapseResult) searcher.cacheLookup("collapseCache", cacheKey); + long indexVersion = searcher.getReader().getVersion(); + if (collapseResult != null) + if (collapseResult.readerVersion == indexVersion) + return; + + collapseResult = new CollapseResult(indexVersion); + + // Do the job + DocList docList = null; + if (collapseParams.collapseType == CollapseType.adjacent) + docList = searcher.getDocList(query, filters, sort, 0, searcher.maxDoc(), flags); + else + docList = searcher.getDocList(query, filters, getSort(sort), 0, searcher.maxDoc(), flags); + + String[] fields = FieldCache.DEFAULT.getStrings(searcher.getReader(), collapseParams.collapseField); + + String previousField = null; + int repeatCount = 0; + int lastVisibleId = -1; + DocIterator it = docList.iterator(); + int collapseCount = 0; + int collapseId = -1; + while (it.hasNext()) { + int currentId = it.nextDoc(); + String currentField = fields[currentId]; + // By default, the current document will be added to the list + boolean bAdd = true; + if (currentField != null && currentField.equals(previousField)) { + if (++repeatCount >= collapseParams.collapseMax) { + // Document wil be collapsed + bAdd = false; + collapseCount++; + if (collapseId == -1) { + collapseResult.hasMoreResult.add(lastVisibleId); + collapseId = lastVisibleId; + } + } + } else { // !currentField.equals + repeatCount = 0; + if (collapseId != -1) { + collapseResult.collapseList.put(new Integer(collapseId), new Integer(collapseCount)); + collapseCount = 0; + collapseId = -1; + } + } + if (bAdd) { + collapseResult.docSet.add(currentId); + lastVisibleId = currentId; + } + previousField = currentField; + } // while + + if (collapseId != -1) + collapseResult.collapseList.put(new Integer(collapseId), new Integer(collapseCount)); + + // Put the result on the cache + searcher.cacheInsert("collapseCache", cacheKey, collapseResult); + } + + /** + * + * @param sort + * @return + */ + private Sort getSort(Sort sort) { + if (sort == null) + return new Sort(new SortField(collapseParams.collapseField)); + SortField[] sf1 = sort.getSort(); + SortField[] sf2 = new SortField[sf1.length + 1]; + for (int i = 0; i < sf1.length; i++) + sf2[i + 1] = sf1[i]; + sf2[0] = new SortField(collapseParams.collapseField); + return new Sort(sf2); + } + + // Get the final docSet + public DocSet getDocSet() { + return collapseResult.docSet; + } + + /** + * Contains document count per group + */ + private NamedList namedList = null; + + /** + * + * @param searcher + * @param docList + * @param params + * @return + * @throws IOException + */ + public NamedList getMoreResults(SolrIndexSearcher searcher, DocList docList) throws IOException { + + if (namedList != null) + return namedList; + + NamedList res = new NamedList(); + DocIterator it = docList.iterator(); + int id = -1; + String[] fields = FieldCache.DEFAULT.getStrings(searcher.getReader(), collapseParams.collapseField); + while (it.hasNext()) { + id = it.nextDoc(); + if (collapseResult.hasMoreResult.exists(id)) + res.add(fields[id], collapseResult.collapseList.get(new Integer(id))); + } + + namedList = res; + return namedList; + } +} Index: src/java/org/apache/solr/search/SolrIndexSearcher.java =================================================================== --- src/java/org/apache/solr/search/SolrIndexSearcher.java (revision 538763) +++ src/java/org/apache/solr/search/SolrIndexSearcher.java (working copy) @@ -653,7 +653,13 @@ return answer.docList; } + public DocList getDocList(Query query, List filterList, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException { + DocListAndSet answer = new DocListAndSet(); + getDocListC(answer,query,filterList,filter,lsort,offset,len,flags); + return answer.docList; + } + private static final int NO_CHECK_QCACHE = 0x80000000; private static final int GET_DOCSET = 0x40000000; private static final int NO_CHECK_FILTERCACHE = 0x20000000; @@ -1188,6 +1194,12 @@ return ret; } + public DocListAndSet getDocListAndSet(Query query, List filterList, DocSet filter, Sort lsort, int offset, int len, int flags) throws IOException { + DocListAndSet ret = new DocListAndSet(); + getDocListC(ret,query,filterList,filter,lsort,offset,len, flags |= GET_DOCSET); + return ret; +} + /** * Returns documents matching both query and filter * and sorted by sort. Also returns the compete set of documents Index: src/java/org/apache/solr/request/StandardRequestHandler.java =================================================================== --- src/java/org/apache/solr/request/StandardRequestHandler.java (revision 538763) +++ src/java/org/apache/solr/request/StandardRequestHandler.java (working copy) @@ -125,13 +125,21 @@ List filters = U.parseFilterQueries(req); SolrIndexSearcher s = req.getSearcher(); + // CollapseFilter is a docset used as intersection + CollapseFilter collapseFilter = null; + DocSet collapseFilterDocSet = null; + if (p.get(COLLAPSE_FIELD) != null) { + collapseFilter = new CollapseFilter(s, query, filters, sort, flags, p); + collapseFilterDocSet = collapseFilter.getDocSet(); + } + if (p.getBool(FACET,false)) { - results = s.getDocListAndSet(query, filters, sort, + results = s.getDocListAndSet(query, filters, collapseFilterDocSet, sort, p.getInt(START,0), p.getInt(ROWS,10), flags); facetInfo = getFacetInfo(req, rsp, results.docSet); } else { - results.docList = s.getDocList(query, filters, sort, + results.docList = s.getDocList(query, filters, collapseFilterDocSet, sort, p.getInt(START,0), p.getInt(ROWS,10), flags); } @@ -141,6 +149,9 @@ rsp.add("response",results.docList); + if (null != collapseFilter) rsp.add("collapse_counts", + collapseFilter.getMoreResults(s, results.docList)); + if (null != facetInfo) rsp.add("facet_counts", facetInfo); try { Index: src/java/org/apache/solr/request/SolrParams.java =================================================================== --- src/java/org/apache/solr/request/SolrParams.java (revision 538763) +++ src/java/org/apache/solr/request/SolrParams.java (working copy) @@ -69,6 +69,22 @@ public static final String HIGHLIGHT_FORMATTER_CLASS = "highlightFormatterClass"; /** + * "normal" or "adjacent" + */ + public static final String COLLAPSE_TYPE = "collapse.type"; + + /** + * Any field whose terms the user wants to enumerate over for Collapse Contraint + */ + public static final String COLLAPSE_FIELD = "collapse.field"; + + /** + * + */ + public static final String COLLAPSE_MAX = "collapse.max"; + + + /** * Should facet counts be calculated? */ public static final String FACET = "facet"; Index: example/solr/conf/solrconfig.xml =================================================================== --- example/solr/conf/solrconfig.xml (revision 538763) +++ example/solr/conf/solrconfig.xml (working copy) @@ -137,6 +137,13 @@ initialSize="512" autowarmCount="0"/> + + + false