Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (revision 1714972) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/QueryImpl.java (working copy) @@ -54,6 +54,7 @@ import org.apache.jackrabbit.oak.query.ast.DescendantNodeJoinConditionImpl; import org.apache.jackrabbit.oak.query.ast.DynamicOperandImpl; import org.apache.jackrabbit.oak.query.ast.EquiJoinConditionImpl; +import org.apache.jackrabbit.oak.query.ast.FacetImpl; import org.apache.jackrabbit.oak.query.ast.FullTextSearchImpl; import org.apache.jackrabbit.oak.query.ast.FullTextSearchScoreImpl; import org.apache.jackrabbit.oak.query.ast.InImpl; @@ -306,6 +307,13 @@ } @Override + public boolean visit(FacetImpl node) { + node.setQuery(query); + node.bindSelector(source); + return super.visit(node); + } + + @Override public boolean visit(FullTextSearchScoreImpl node) { node.setQuery(query); node.bindSelector(source); @@ -1112,7 +1120,7 @@ if (path == null) { return null; } - if (!JcrPathParser.validate(path)) { + if (!JcrPathParser.validate(path) && !path.startsWith("facet(")) { // TODO : fix validation to allow facet(...) throw new IllegalArgumentException("Invalid path: " + path); } String p = namePathMapper.getOakPath(path); Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitor.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitor.java (revision 1714972) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitor.java (working copy) @@ -86,4 +86,6 @@ boolean visit(SpellcheckImpl node); boolean visit(SuggestImpl suggest); + + boolean visit(FacetImpl facet); } \ No newline at end of file Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitorBase.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitorBase.java (revision 1714972) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/AstVisitorBase.java (working copy) @@ -102,6 +102,15 @@ } /** + * Calls accept on the static operand in the facet constraint. + */ + @Override + public boolean visit(FacetImpl node) { + node.getExpression().accept(this); + return true; + } + + /** * Calls accept on the two sources and the join condition in the join node. */ @Override Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FacetImpl.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FacetImpl.java (revision 0) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FacetImpl.java (working copy) @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.query.ast; + +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.query.index.FilterImpl; +import org.apache.jackrabbit.oak.spi.query.PropertyValues; +import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.Set; + +/** + * Support for "facet(...) + */ +public class FacetImpl extends ConstraintImpl { + + private final Logger log = LoggerFactory.getLogger(getClass()); + + public static final String NATIVE_LUCENE_LANGUAGE = "lucene"; + + public static final String FACET_PREFIX = "facet?fields="; + + private final String selectorName; + private final StaticOperandImpl expression; + private SelectorImpl selector; + + FacetImpl(String selectorName, StaticOperandImpl expression) { + this.selectorName = selectorName; + this.expression = expression; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("facet("); + builder.append(quote(selectorName)); + builder.append(", "); + builder.append(getExpression()); + builder.append(')'); + return builder.toString(); + } + + @Override + public boolean evaluate() { + // disable evaluation if a fulltext index is used, + // and because we don't know how to process native + // conditions + if (!(selector.getIndex() instanceof FulltextQueryIndex)) { + log.warn("No full-text index was found that can process the condition " + toString()); + return false; + } + // we assume the index only returns the requested entries + return true; + } + + @Override + public Set getPropertyExistenceConditions() { + return Collections.emptySet(); + } + + @Override + public void restrict(FilterImpl f) { + if (f.getSelector().equals(selector)) { + PropertyValue p = expression.currentValue(); + String term = p.getValue(Type.STRING); + String query = FACET_PREFIX + term; + PropertyValue v = PropertyValues.newString(query); + f.restrictProperty(NativeFunctionImpl.NATIVE_PREFIX + NATIVE_LUCENE_LANGUAGE, Operator.EQUAL, v); + } + } + + @Override + public void restrictPushDown(SelectorImpl s) { + if (s.equals(selector)) { + selector.restrictSelector(this); + } + } + + @Override + public Set getSelectors() { + return Collections.emptySet(); + } + + @Override + boolean accept(AstVisitor v) { + return v.visit(this); + } + + public void bindSelector(SourceImpl source) { + selector = source.getExistingSelector(selectorName); + } + + public StaticOperandImpl getExpression() { + return expression; + } + +} \ No newline at end of file Property changes on: oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/FacetImpl.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java =================================================================== --- oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java (revision 1714972) +++ oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java (working copy) @@ -70,7 +70,7 @@ */ public class SelectorImpl extends SourceImpl { private static final Logger LOG = LoggerFactory.getLogger(SelectorImpl.class); - + // TODO possibly support using multiple indexes (using index intersection / index merge) private SelectorExecutionPlan plan; @@ -665,6 +665,8 @@ result = currentRow.getValue(QueryImpl.REP_SPELLCHECK); } else if (oakPropertyName.equals(QueryImpl.REP_SUGGEST)) { result = currentRow.getValue(QueryImpl.REP_SUGGEST); + } else if (oakPropertyName.startsWith("facet(")) { + result = currentRow.getValue(oakPropertyName); } else { result = PropertyValues.create(t.getProperty(oakPropertyName)); } Index: oak-lucene/pom.xml =================================================================== --- oak-lucene/pom.xml (revision 1714972) +++ oak-lucene/pom.xml (working copy) @@ -218,6 +218,12 @@ lucene-highlighter ${lucene.version} + + org.apache.lucene + lucene-facet + ${lucene.version} + provided + Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (revision 1714972) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (working copy) @@ -119,4 +119,16 @@ } return FULLTEXT_RELATIVE_NODE + nodeRelativePath; } + + /** + * Encodes the field name such that it can be used for faceting. + * This is done such a field if used for both faceting and querying uses + * a different name for facet field + * + * @param name name to encode + * @return encoded field name + */ + public static String createFacetFieldName(String name) { + return "facet:" + name; + } } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (revision 1714972) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (working copy) @@ -284,4 +284,9 @@ * existing index files */ String INDEX_PATH = "indexPath"; + + /** + * Optional property indicating facet can be retrieved together with plain queries. + */ + String PROP_FACET = "facet"; } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (revision 1714972) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (working copy) @@ -69,6 +69,7 @@ import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; +import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.util.BytesRef; @@ -335,6 +336,10 @@ dirty |= addTypedOrderedFields(fields, property, pname, pd); } + if (pd.facet) { + dirty |= addFacetFields(fields, property, pname, pd); + } + dirty |= indexProperty(path, fields, state, property, pname, pd); } @@ -399,6 +404,59 @@ return document; } + private boolean addFacetFields(List fields, PropertyState property, String pname, PropertyDefinition pd) { + if (property.getType().isArray()) { + log.warn( + "[{}] Ignoring ordered property {} of type {} for path {} as multivalued facets are not supported", + getIndexName(), pname, + Type.fromTag(property.getType().tag(), true), getPath()); + return false; + } + + int tag = property.getType().tag(); + int idxDefinedTag = pd.getType(); + // Try converting type to the defined type in the index definition + if (tag != idxDefinedTag) { + log.debug( + "[{}] Facet property defined with type {} differs from property {} with type {} in " + + "path {}", + getIndexName(), + Type.fromTag(idxDefinedTag, false), property.toString(), + Type.fromTag(tag, false), getPath()); + tag = idxDefinedTag; + } + + String name = FieldNames.createFacetFieldName(pname); + boolean fieldAdded = false; + Field f = null; + try { + if (tag == Type.LONG.tag()) { + f = new SortedSetDocValuesFacetField(name, property.getValue(Type.LONG).toString()); + } else if (tag == Type.DATE.tag()) { + String date = property.getValue(Type.DATE); + f = new SortedSetDocValuesFacetField(name, date); + } else if (tag == Type.DOUBLE.tag()) { + f = new DoubleDocValuesField(name, property.getValue(Type.DOUBLE)); + } else if (tag == Type.BOOLEAN.tag()) { + f = new SortedSetDocValuesFacetField(name, property.getValue(Type.BOOLEAN).toString()); + } else if (tag == Type.STRING.tag()) { + f = new SortedSetDocValuesFacetField(name, property.getValue(Type.STRING)); + } + + if (f != null) { + fields.add(f); + fieldAdded = true; + } + } catch (Exception e) { + log.warn( + "[{}] Ignoring facet property. Could not convert property {} of type {} to type {} for path {}", + getIndexName(), pname, + Type.fromTag(property.getType().tag(), false), + Type.fromTag(tag, false), getPath(), e); + } + return fieldAdded; + } + private boolean indexProperty(String path, List fields, NodeState state, Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (revision 1714972) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (working copy) @@ -31,6 +31,8 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterables; @@ -71,6 +73,11 @@ import org.apache.lucene.analysis.CachingTokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; +import org.apache.lucene.facet.FacetResult; +import org.apache.lucene.facet.Facets; +import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.sortedset.DefaultSortedSetDocValuesReaderState; +import org.apache.lucene.facet.sortedset.SortedSetDocValuesFacetCounts; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexReader; @@ -187,6 +194,8 @@ */ static final int LUCENE_QUERY_BATCH_SIZE = 50; + static final Pattern FACET_REGEX = Pattern.compile("facet\\((\\w+(\\:\\w+)?)\\)"); + protected final IndexTracker tracker; private final ScorerProviderFactory scorerProviderFactory; @@ -301,7 +310,7 @@ return endOfData(); } - private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, String excerpt) throws IOException { + private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, String excerpt, Facets facets) throws IOException { IndexReader reader = searcher.getIndexReader(); //TODO Look into usage of field cache for retrieving the path //instead of reading via reader if no of docs in index are limited @@ -330,7 +339,7 @@ } LOG.trace("Matched path {}", path); - return new LuceneResultRow(path, doc.score, excerpt); + return new LuceneResultRow(path, doc.score, excerpt, facets); } return null; } @@ -363,6 +372,14 @@ checkForIndexVersionChange(searcher); + String facetField = null; + if (plan.getFilter().getQueryStatement() != null && plan.getFilter().getQueryStatement().contains("facet(")) { + Matcher matcher = FACET_REGEX.matcher(plan.getFilter().getQueryStatement()); + if (matcher.find()) { + facetField = FieldNames.createFacetFieldName(matcher.group(1)); + } + } + TopDocs docs; long start = PERF_LOGGER.start(); while (true) { @@ -384,6 +401,14 @@ PERF_LOGGER.end(start, -1, "{} ...", docs.scoreDocs.length); nextBatchSize = (int) Math.min(nextBatchSize * 2L, 100000); + Facets facets = null; + if (facetField != null) { + DefaultSortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader(), facetField); + FacetsCollector facetsCollector = new FacetsCollector(); + FacetsCollector.search(searcher, query, 10, facetsCollector); + facets = new SortedSetDocValuesFacetCounts(state, facetsCollector); + } + boolean addExcerpt = filter.getQueryStatement() != null && filter.getQueryStatement().contains(QueryImpl.REP_EXCERPT); for (ScoreDoc doc : docs.scoreDocs) { String excerpt = null; @@ -391,7 +416,7 @@ excerpt = getExcerpt(indexNode, searcher, query, doc); } - LuceneResultRow row = convertToRow(doc, searcher, excerpt); + LuceneResultRow row = convertToRow(doc, searcher, excerpt, facets); if (row != null) { queue.add(row); } @@ -1326,9 +1351,11 @@ final Iterable suggestWords; final boolean isVirutal; final String excerpt; + final Facets facets; - LuceneResultRow(String path, double score, String excerpt) { + LuceneResultRow(String path, double score, String excerpt, Facets facets) { this.excerpt = excerpt; + this.facets = facets; this.isVirutal = false; this.path = path; this.score = score; @@ -1341,6 +1368,7 @@ this.score = 1.0d; this.suggestWords = suggestWords; this.excerpt = null; + this.facets = null; } @Override @@ -1429,6 +1457,25 @@ if (QueryImpl.REP_EXCERPT.equals(columnName)) { return PropertyValues.newString(currentRow.excerpt); } + if (columnName.startsWith("facet(")) { + Matcher m = FACET_REGEX.matcher(columnName); + if (m.matches()) { + String facetFieldName = m.group(1); + Facets facets = currentRow.facets; + try { + if (facets != null) { + FacetResult topChildren = facets.getTopChildren(10, facetFieldName); + if (topChildren != null) { + return PropertyValues.newString(topChildren.toString()); + } else { + return null; + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } return pathRow.getValue(columnName); } Index: oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java =================================================================== --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (revision 1714972) +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (working copy) @@ -86,6 +86,8 @@ final boolean useInSpellcheck; + final boolean facet; + final String[] ancestors; /** @@ -124,6 +126,7 @@ this.propertyType = getPropertyType(idxDefn, nodeName, defn); this.useInSuggest = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_USE_IN_SUGGEST, false); this.useInSpellcheck = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, false); + this.facet = getOptionalValue(defn, LuceneIndexConstants.PROP_FACET, false); this.nullCheckEnabled = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_NULL_CHECK_ENABLED, false); this.notNullCheckEnabled = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_NOT_NULL_CHECK_ENABLED, false); this.nonRelativeName = determineNonRelativeName(); Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (revision 1714972) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (working copy) @@ -106,6 +106,7 @@ .setProperty(LuceneIndexConstants.PROP_PROPERTY_INDEX, true) .setProperty(LuceneIndexConstants.PROP_USE_IN_SPELLCHECK, true) .setProperty(LuceneIndexConstants.PROP_USE_IN_SUGGEST, true) + .setProperty(LuceneIndexConstants.PROP_FACET, true) .setProperty(LuceneIndexConstants.PROP_NAME, LuceneIndexConstants.REGEX_ALL_PROPS) .setProperty(LuceneIndexConstants.PROP_IS_REGEX, true); } Index: oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java =================================================================== --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (revision 0) +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (working copy) @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.jcr.query; + +import org.apache.jackrabbit.core.query.AbstractQueryTest; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.Value; +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import javax.jcr.query.RowIterator; + +/** + * Test for faceting capabilities from the JCR spec point of view + */ +public class FacetTest extends AbstractQueryTest { + + public void testFacetRetrieval() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("text", "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("text", "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("text", "oh hallo"); + session.save(); + + String sql2 = "select [jcr:path], [facet(text)] from [nt:base] " + + "where contains([text], 'hello OR hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "text:[hallo (2), hello (1), oh (1)]"; + assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "facet(text)")); + } + + public void testFacetRetrieval4() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("jcr:title", "apache jackrabbit oak"); + n1.setProperty("tags", new String[]{"software", "repository", "apache"}); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("jcr:title", "oak furniture"); + n2.setProperty("tags", "furniture"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("jcr:title", "oak cosmetics"); + n3.setProperty("tags", "cosmetics"); + Node n4 = testRootNode.addNode("node4"); + n4.setProperty("jcr:title", "oak and aem"); + n4.setProperty("tags", new String[]{"software", "repository", "aem"}); + session.save(); + + String sql2 = "select [jcr:path], [facet(tags)] from [nt:base] " + + "where contains([jcr:title], 'oak') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)]"; + assertEquals(facetResult, getResult(result, "facet(tags)")); + } + + public void testFacetRetrievalWithAnonymousUser() throws Exception { + Session session = superuser; + + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("text", "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("text", "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("text", "oh hallo"); + session.save(); + + session = getHelper().getReadOnlySession(); + QueryManager qm = session.getWorkspace().getQueryManager(); + + String sql2 = "select [jcr:path], [facet(text)] from [nt:base] " + + "where contains([text], 'hello OR hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "text:[hallo (2), hello (1), oh (1)]"; + assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "facet(text)")); + } + + public void testFacetRetrieval2() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + String pn = "jcr:title"; + n1.setProperty(pn, "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty(pn, "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty(pn, "oh hallo"); + session.save(); + + String sql2 = "select [jcr:path], [facet(" + pn + ")] from [nt:base] " + + "where contains([" + pn + "], 'hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = pn + ":[hallo (2), oh (1)]"; + assertEquals(facetResult + ", " + facetResult, getResult(result, "facet(" + pn + ")")); + } + + public void testFacetRetrieval3() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + String pn = "jcr:title"; + String pn2 = "jcr:description"; + n1.setProperty(pn, "hello"); + n1.setProperty(pn2, "a"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty(pn, "hallo"); + n2.setProperty(pn2, "b"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty(pn, "oh hallo"); + n3.setProperty(pn2, "a"); + session.save(); + + String sql2 = "select [jcr:path], [facet(" + pn + ")], [facet(" + pn2 + ")] from [nt:base] " + + "where contains([" + pn + "], 'hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = pn + ":[hallo (2), oh (1)], " + pn2 + ":[a (1), b (1)], " + pn + ":[hallo (2), oh (1)], " + pn2 + ":[a (1), b (1)]"; + assertEquals(facetResult, getResult(result, "facet(" + pn + ")", "facet(" + pn2 + ")")); + } + + static String getResult(QueryResult result, String... propertyNames) throws RepositoryException { + StringBuilder buff = new StringBuilder(); + RowIterator it = result.getRows(); + while (it.hasNext()) { + + Row row = it.nextRow(); + for (String propertyName : propertyNames) { + Value value = row.getValue(propertyName); + if (value != null) { + if (buff.length() > 0) { + buff.append(", "); + } + buff.append(value.getString()); + } + } + } + return buff.toString(); + } + +} \ No newline at end of file Property changes on: oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java =================================================================== --- oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java (revision 1714972) +++ oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/FilterQueryParser.java (working copy) @@ -17,7 +17,10 @@ package org.apache.jackrabbit.oak.plugins.index.solr.query; import java.util.Collection; +import java.util.LinkedList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration; import org.apache.jackrabbit.oak.query.QueryImpl; @@ -29,10 +32,12 @@ import org.apache.jackrabbit.oak.query.fulltext.FullTextVisitor; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.query.QueryIndex; +import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.solr.client.solrj.SolrQuery; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; import static org.apache.jackrabbit.oak.plugins.index.solr.util.SolrUtils.getSortingField; import static org.apache.jackrabbit.oak.plugins.index.solr.util.SolrUtils.partialEscape; @@ -44,6 +49,7 @@ class FilterQueryParser { private static final Logger log = LoggerFactory.getLogger(FilterQueryParser.class); + static final Pattern FACET_REGEX = Pattern.compile("facet\\((\\w+(\\:\\w+)?)\\)"); static SolrQuery getQuery(Filter filter, List sortOrder, OakSolrConfiguration configuration) { @@ -63,6 +69,21 @@ } } + // facet enable + String queryStatement = filter.getQueryStatement(); + if (queryStatement != null) { + Matcher matcher = FACET_REGEX.matcher(queryStatement); + int start = 0; + while (matcher.find(start)) { + String facetField = matcher.group(1); + solrQuery.addFacetField(facetField); + start = matcher.end(); + } + if (start > 0) { + solrQuery.setFacetMinCount(1); + } + } + if (sortOrder != null) { for (QueryIndex.OrderEntry orderEntry : sortOrder) { SolrQuery.ORDER order; @@ -367,4 +388,53 @@ return partialEscape(filter.getPath()).toString(); } + private static void addACLPathsFilterQuery(SolrQuery query, NodeState root, OakSolrConfiguration configuration) { + StringBuilder stringBuilder = new StringBuilder(); + + // add path exact filters + Collection nodes = getNodes(root, "", -1); + for (String p : nodes) { + if (stringBuilder.length() > 0) { + stringBuilder.append(" OR "); + } + stringBuilder.append(partialEscape(p)); + } + + query.addFilterQuery(configuration.getFieldForPathRestriction( + Filter.PathRestriction.EXACT) + ":(" + stringBuilder.toString() + ")"); + + // add all children filters +// stringBuilder = new StringBuilder(); +// for (String p : collectReadableRoots(root)) { +// if (stringBuilder.length() > 0) { +// stringBuilder.append(" OR "); +// } +// stringBuilder.append(partialEscape(p)); +// } +// query.addFilterQuery(configuration.getFieldForPathRestriction(Filter.PathRestriction.ALL_CHILDREN) + +// ":(" + stringBuilder.toString() + ")"); + } + + private static Collection getNodes(NodeState nodeState, String path, int depth) { + Collection paths = new LinkedList(); + if (depth != 0) { + for (String name : nodeState.getChildNodeNames()) { + NodeState child = nodeState.getChildNode(checkNotNull(name)); + String childPath = path + "/" + name; + paths.add(childPath); + paths.addAll(getNodes(child, childPath, depth - 1)); + } + } + return paths; + } + + private static Collection collectReadableRoots(NodeState root) { + // TODO : this should build a flat list of paths at a certain depth from the passed node state + Collection paths = new LinkedList(); + for (String childName : root.getChildNodeNames()) { + paths.add("/" + childName); + } + return paths; + } + } Index: oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java =================================================================== --- oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java (revision 1714972) +++ oak-solr-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrQueryIndex.java (working copy) @@ -17,15 +17,18 @@ package org.apache.jackrabbit.oak.plugins.index.solr.query; import javax.annotation.CheckForNull; +import javax.jcr.Session; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; import com.google.common.collect.AbstractIterator; import com.google.common.collect.Iterables; @@ -33,6 +36,7 @@ import com.google.common.collect.Sets; import org.apache.jackrabbit.oak.api.PropertyValue; import org.apache.jackrabbit.oak.api.Result.SizePrecision; +import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.solr.configuration.OakSolrConfiguration; import org.apache.jackrabbit.oak.query.QueryEngineSettings; @@ -48,11 +52,13 @@ import org.apache.jackrabbit.oak.spi.query.QueryConstants; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex; +import org.apache.jackrabbit.oak.spi.security.authorization.permission.PermissionProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServer; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; +import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.SpellCheckResponse; import org.apache.solr.common.SolrDocument; @@ -226,7 +232,7 @@ final int parentDepth = getDepth(parent); - AbstractIterator iterator = getIterator(filter, sortOrder, parent, parentDepth); + AbstractIterator iterator = getIterator(filter, sortOrder, parent, parentDepth, root); cursor = new SolrRowCursor(iterator, plan, filter.getQueryEngineSettings()); } catch (Exception e) { @@ -235,8 +241,10 @@ return cursor; } - private AbstractIterator getIterator(final Filter filter, final List sortOrder, final String parent, final int parentDepth) { + private AbstractIterator getIterator(final Filter filter, final List sortOrder, final String parent, + final int parentDepth, final NodeState root) { return new AbstractIterator() { + public Collection facetFields = new LinkedList(); private final Set seenPaths = Sets.newHashSet(); private final Deque queue = Queues.newArrayDeque(); private int offset = 0; @@ -270,7 +278,7 @@ if (scoreObj != null) { score = (Float) scoreObj; } - return new SolrResultRow(path, score, doc); + return new SolrResultRow(path, score, doc, facetFields); } @@ -343,12 +351,39 @@ } } + // get facets + List returnedFieldFacet = queryResponse.getFacetFields(); + if (returnedFieldFacet != null) { + facetFields.addAll(returnedFieldFacet); + } + + // filter facets on doc paths + if (!facetFields.isEmpty()) { + for (SolrDocument doc : docs) { + String path = String.valueOf(doc.getFieldValue(configuration.getPathField())); + // if path doesn't exist in the node state, filter the facets + PermissionProvider permissionProvider = filter.getSelector().getQuery().getExecutionContext().getPermissionProvider(); + for (FacetField ff : facetFields) { + if (permissionProvider != null) { + if (!permissionProvider.isGranted(path+"/"+ff.getName(), Session.ACTION_READ)) { + filterFacet(doc, ff); + } + } else { // fallback in case of missing PermissionProvider + if (!exists(path, root)) { // this will only work in case the NodeState is a SecureNodeState + filterFacet(doc, ff); + } + } + } + + } + } + // handle spellcheck SpellCheckResponse spellCheckResponse = queryResponse.getSpellCheckResponse(); if (spellCheckResponse != null && spellCheckResponse.getSuggestions() != null && spellCheckResponse.getSuggestions().size() > 0) { SolrDocument fakeDoc = getSpellChecks(spellCheckResponse, filter); - queue.add(new SolrResultRow("/", 1.0, fakeDoc)); + queue.add(new SolrResultRow("/", 1.0, fakeDoc, Collections.emptyList())); noDocs = true; } @@ -359,7 +394,7 @@ Set> suggestEntries = suggest.entrySet(); if (!suggestEntries.isEmpty()) { SolrDocument fakeDoc = getSuggestions(suggestEntries, filter); - queue.add(new SolrResultRow("/", 1.0, fakeDoc)); + queue.add(new SolrResultRow("/", 1.0, fakeDoc, Collections.emptyList())); noDocs = true; } } @@ -376,6 +411,52 @@ }; } + private void filterFacet(SolrDocument doc, FacetField facetField) { + // TODO : facet filtering by value requires that the facet values match the stored values + // TODO : a *_facet field must exist, storing docValues instead of values and that should be used for faceting and at filtering time + if (doc.getFieldNames().contains(facetField.getName())) { + // decrease facet value + Collection docFieldValues = doc.getFieldValues(facetField.getName()); + if (docFieldValues != null) { + for (Object docFieldValue : docFieldValues) { + String valueString = String.valueOf(docFieldValue); + List toRemove = new LinkedList(); + for (FacetField.Count count : facetField.getValues()) { + long existingCount = count.getCount(); + if (valueString.equals(count.getName())) { + if (existingCount > 1) { + // decrease the count + count.setCount(existingCount - 1); + } else { + // remove the entire entry + toRemove.add(count); + } + } + } + for (FacetField.Count f : toRemove) { + assert facetField.getValues().remove(f); + } + } + } + } + } + + private boolean exists(String path, NodeState root) { + // need to enable the check at the property level too + boolean nodeExists = true; + NodeState nodeState = root; + for (String n : PathUtils.elements(path)) { + if (nodeState.hasChildNode(n)) { + nodeState = nodeState.getChildNode(n); + } else { + nodeExists = false; + break; + } + } + return nodeExists; + } + + private SolrDocument getSpellChecks(SpellCheckResponse spellCheckResponse, Filter filter) throws SolrServerException { SolrDocument fakeDoc = new SolrDocument(); List suggestions = spellCheckResponse.getSuggestions(); @@ -500,11 +581,13 @@ final String path; final double score; final SolrDocument doc; + final Collection facetFields; - SolrResultRow(String path, double score, SolrDocument doc) { + SolrResultRow(String path, double score, SolrDocument doc, Collection facetFields) { this.path = path; this.score = score; this.doc = doc; + this.facetFields = facetFields; } @Override @@ -581,6 +664,24 @@ if (QueryImpl.JCR_SCORE.equals(columnName)) { return PropertyValues.newDouble(currentRow.score); } + if (columnName.startsWith("facet(")) { + Matcher m = FilterQueryParser.FACET_REGEX.matcher(columnName); + if (m.matches()) { // facets + String facetFieldName = m.group(1); + FacetField facetField = null; + for (FacetField ff : currentRow.facetFields) { + if (ff.getName().equals(facetFieldName)) { + facetField = ff; + break; + } + } + if (facetField != null) { + return PropertyValues.newString(facetField.toString()); + } else { + return null; + } + } + } Collection fieldValues = currentRow.doc.getFieldValues(columnName); String value; if (fieldValues != null && fieldValues.size() > 0) { Index: oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java =================================================================== --- oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (revision 0) +++ oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java (working copy) @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.jcr.query; + +import javax.jcr.Node; +import javax.jcr.RepositoryException; +import javax.jcr.Session; +import javax.jcr.SimpleCredentials; +import javax.jcr.Value; +import javax.jcr.query.Query; +import javax.jcr.query.QueryManager; +import javax.jcr.query.QueryResult; +import javax.jcr.query.Row; +import javax.jcr.query.RowIterator; + +import org.apache.jackrabbit.core.query.AbstractQueryTest; + +/** + * Test for faceting capabilities from the JCR spec point of view + */ +public class FacetTest extends AbstractQueryTest { + + public void testFacetRetrieval() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("text", "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("text", "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("text", "oh hallo"); + session.save(); + + String sql2 = "select [jcr:path], [facet(text)] from [nt:base] " + + "where contains([text], 'hello OR hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "text:[hallo (2), hello (1), oh (1)]"; + assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "facet(text)")); + } + + public void testFacetRetrieval4() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("jcr:title", "apache jackrabbit oak"); + n1.setProperty("tags", new String[]{"software", "repository", "apache"}); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("jcr:title", "oak furniture"); + n2.setProperty("tags", "furniture"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("jcr:title", "oak cosmetics"); + n3.setProperty("tags", "cosmetics"); + Node n4 = testRootNode.addNode("node4"); + n4.setProperty("jcr:title", "oak and aem"); + n4.setProperty("tags", new String[]{"software", "repository", "aem"}); + session.save(); + + String sql2 = "select [jcr:path], [facet(tags)] from [nt:base] " + + "where contains([jcr:title], 'oak') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)], tags:[repository (2), software (2), aem (1), apache (1), cosmetics (1), furniture (1)]"; + assertEquals(facetResult, getResult(result, "facet(tags)")); + } + + public void testFacetRetrievalWithAnonymousUser() throws Exception { + Session session = superuser; + + Node n1 = testRootNode.addNode("node1"); + n1.setProperty("text", "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty("text", "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty("text", "oh hallo"); + session.save(); + + session = getHelper().getReadOnlySession(); + QueryManager qm = session.getWorkspace().getQueryManager(); + + String sql2 = "select [jcr:path], [facet(text)] from [nt:base] " + + "where contains([text], 'hello OR hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = "text:[hallo (2), hello (1), oh (1)]"; + assertEquals(facetResult + ", " + facetResult + ", " + facetResult, getResult(result, "facet(text)")); + } + + public void testFacetRetrieval2() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + String pn = "jcr:title"; + n1.setProperty(pn, "hello"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty(pn, "hallo"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty(pn, "oh hallo"); + session.save(); + + String sql2 = "select [jcr:path], [facet(" + pn + ")] from [nt:base] " + + "where contains([" + pn + "], 'hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = pn + ":[hallo (2), oh (1)]"; + assertEquals(facetResult + ", " + facetResult, getResult(result, "facet(" + pn + ")")); + } + + public void testFacetRetrieval3() throws Exception { + Session session = superuser; + QueryManager qm = session.getWorkspace().getQueryManager(); + Node n1 = testRootNode.addNode("node1"); + String pn = "jcr:title"; + String pn2 = "jcr:description"; + n1.setProperty(pn, "hello"); + n1.setProperty(pn2, "a"); + Node n2 = testRootNode.addNode("node2"); + n2.setProperty(pn, "hallo"); + n2.setProperty(pn2, "b"); + Node n3 = testRootNode.addNode("node3"); + n3.setProperty(pn, "oh hallo"); + n3.setProperty(pn2, "a"); + session.save(); + + String sql2 = "select [jcr:path], [facet(" + pn + ")], [facet(" + pn2 + ")] from [nt:base] " + + "where contains([" + pn + "], 'hallo') order by [jcr:path]"; + Query q = qm.createQuery(sql2, Query.JCR_SQL2); + QueryResult result = q.execute(); + String facetResult = pn + ":[hallo (2), oh (1)], " + pn2 + ":[a (1), b (1)], " + pn + ":[hallo (2), oh (1)], " + pn2 + ":[a (1), b (1)]"; + assertEquals(facetResult, getResult(result, "facet(" + pn + ")", "facet(" + pn2 + ")")); + } + + static String getResult(QueryResult result, String... propertyNames) throws RepositoryException { + StringBuilder buff = new StringBuilder(); + RowIterator it = result.getRows(); + while (it.hasNext()) { + + Row row = it.nextRow(); + for (String propertyName : propertyNames) { + Value value = row.getValue(propertyName); + if (value != null) { + if (buff.length() > 0) { + buff.append(", "); + } + buff.append(value.getString()); + } + } + } + return buff.toString(); + } + +} \ No newline at end of file Property changes on: oak-solr-core/src/test/java/org/apache/jackrabbit/oak/jcr/query/FacetTest.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java =================================================================== --- oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java (revision 1714972) +++ oak-solr-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/solr/query/SolrIndexQueryTestIT.java (working copy) @@ -47,7 +47,7 @@ /** * General query extensive testcase for {@link SolrQueryIndex} */ - public class SolrIndexQueryTestIT extends AbstractQueryTest { +public class SolrIndexQueryTestIT extends AbstractQueryTest { @Rule public TestName name = new TestName();