diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java index 6796d4a..fcd5c7e 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java @@ -20,9 +20,11 @@ import java.util.Arrays; import com.google.common.primitives.Ints; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.util.ISO8601; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.IntField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -108,6 +110,14 @@ public final class FieldFactory { return new TextField(FieldNames.createFulltextFieldName(name), value, NO); } + public static Field newAncestorsField(String path){ + return new TextField(FieldNames.ANCESTORS, path, NO); + } + + public static Field newDepthField(String path){ + return new IntField(FieldNames.PATH_DEPTH, PathUtils.getDepth(path), NO); + } + /** * Date values are saved with sec resolution * @param date jcr data string diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java index f7fa07e..4a18b4d 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java @@ -38,6 +38,17 @@ public final class FieldNames { public static final String PATH = ":path"; /** + * Name of the field that contains all the path hierarchy e.g. for /a/b/c + * it would contain /a, /a/b, /a/b/c + */ + public static final String ANCESTORS = ":ancestors"; + + /** + * Name of the field which refers to the depth of path + */ + public static final String PATH_DEPTH = ":depth"; + + /** * Name of the field that contains the fulltext index. */ public static final String FULLTEXT = ":fulltext"; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 74cc845..e0669f2 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.namepath.NamePathMapper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain; import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; import org.apache.jackrabbit.oak.plugins.nodetype.ReadOnlyNodeTypeManager; import org.apache.jackrabbit.oak.plugins.tree.ImmutableTree; @@ -52,6 +53,9 @@ import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.util.TreeUtil; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.miscellaneous.PerFieldAnalyzerWrapper; +import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory; import org.apache.lucene.codecs.Codec; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -153,6 +157,8 @@ class IndexDefinition implements Aggregate.AggregateMapper{ private final boolean indexesAllTypes; + private final Analyzer analyzer; + public IndexDefinition(NodeState root, NodeState defn) { this(root, defn, null); } @@ -206,6 +212,7 @@ class IndexDefinition implements Aggregate.AggregateMapper{ this.costPerEntry = getOptionalValue(defn, LuceneIndexConstants.COST_PER_ENTRY, 1.0); this.costPerExecution = getOptionalValue(defn, LuceneIndexConstants.COST_PER_EXECUTION, 1.0); this.indexesAllTypes = areAllTypesIndexed(); + this.analyzer = createAnalyzer(); } public boolean isFullTextEnabled() { @@ -282,11 +289,28 @@ class IndexDefinition implements Aggregate.AggregateMapper{ return indexesAllTypes; } + public Analyzer getAnalyzer(){ + return analyzer; + } + @Override public String toString() { return "IndexDefinition : " + indexName; } + //~---------------------------------------------------< Analyzer > + + private Analyzer createAnalyzer() { + if (!evaluatePathRestrictions()){ + return LuceneIndexConstants.ANALYZER; + } + Map analyzerMap = ImmutableMap.builder() + .put(FieldNames.ANCESTORS, + new TokenizerChain(new PathHierarchyTokenizerFactory(Collections.emptyMap()))) + .build(); + return new PerFieldAnalyzerWrapper(LuceneIndexConstants.ANALYZER, analyzerMap); + } + //~---------------------------------------------------< Aggregates > @CheckForNull diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java index f6a0d41..cd2ac0f 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java @@ -290,6 +290,7 @@ class IndexPlanner { } private String getPathPrefix() { + // 2 = /oak:index/ String parentPath = PathUtils.getAncestorPath(indexPath, 2); return PathUtils.denotesRoot(parentPath) ? "" : parentPath; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index c3f045a..eac87ab 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -165,15 +165,10 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { protected final IndexTracker tracker; - private final Analyzer analyzer; - private final NodeAggregator aggregator; - public LuceneIndex( - IndexTracker tracker, Analyzer analyzer, - NodeAggregator aggregator) { + public LuceneIndex(IndexTracker tracker, NodeAggregator aggregator) { this.tracker = tracker; - this.analyzer = analyzer; this.aggregator = aggregator; } @@ -247,7 +242,7 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { // we only restrict non-full-text conditions if there is // no relative property in the full-text constraint boolean nonFullTextConstraints = parent.isEmpty(); - String planDesc = getQuery(filter, null, nonFullTextConstraints, analyzer, index.getDefinition()) + " ft:(" + ft + ")"; + String planDesc = getQuery(filter, null, nonFullTextConstraints, index.getDefinition()) + " ft:(" + ft + ")"; if (!parent.isEmpty()) { planDesc += " parent:" + parent; } @@ -333,7 +328,7 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { try { IndexSearcher searcher = indexNode.getSearcher(); Query query = getQuery(filter, searcher.getIndexReader(), - nonFullTextConstraints, analyzer, indexNode.getDefinition()); + nonFullTextConstraints, indexNode.getDefinition()); TopDocs docs; long time = System.currentTimeMillis(); if (lastDoc != null) { @@ -440,8 +435,9 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { * @return the Lucene query */ private static Query getQuery(Filter filter, IndexReader reader, - boolean nonFullTextConstraints, Analyzer analyzer, IndexDefinition indexDefinition) { + boolean nonFullTextConstraints, IndexDefinition indexDefinition) { List qs = new ArrayList(); + Analyzer analyzer = indexDefinition.getAnalyzer(); FullTextExpression ft = filter.getFullTextConstraint(); if (ft == null) { // there might be no full-text constraint @@ -453,7 +449,7 @@ public class LuceneIndex implements AdvanceFulltextQueryIndex { PropertyRestriction pr = filter.getPropertyRestriction(NATIVE_QUERY_FUNCTION); if (pr != null) { String query = String.valueOf(pr.first.getValue(pr.first.getType())); - QueryParser queryParser = new QueryParser(VERSION, "", analyzer); + QueryParser queryParser = new QueryParser(VERSION, "", indexDefinition.getAnalyzer()); if (query.startsWith("mlt?")) { String mltQueryString = query.replace("mlt?", ""); if (reader != null) { diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 96d08f6..0abfd73 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -141,7 +141,7 @@ public interface LuceneIndexConstants { */ String TEST_MODE = "testMode"; - String EVALUATE_PATH_RESTRICTION = "oak.experimental.evaluatePathRestrictions"; + String EVALUATE_PATH_RESTRICTION = "evaluatePathRestrictions"; /** * Experimental config to restrict which property type gets indexed at diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 489b98a..447e41f 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -19,7 +19,9 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.oak.commons.PathUtils.concat; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; +import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newDepthField; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newFulltextField; +import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newAncestorsField; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPathField; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPropertyField; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; @@ -51,7 +53,6 @@ import org.apache.jackrabbit.oak.plugins.tree.ImmutableTree; import org.apache.jackrabbit.oak.spi.commit.Editor; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.DoubleDocValuesField; import org.apache.lucene.document.DoubleField; @@ -108,12 +109,12 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { private final MatcherState matcherState; - LuceneIndexEditor(NodeState root, NodeBuilder definition, Analyzer analyzer, + LuceneIndexEditor(NodeState root, NodeBuilder definition, IndexUpdateCallback updateCallback) throws CommitFailedException { this.parent = null; this.name = null; this.path = "/"; - this.context = new LuceneIndexEditorContext(root, definition, analyzer, + this.context = new LuceneIndexEditorContext(root, definition, updateCallback); this.root = root; this.isDeleted = false; @@ -314,6 +315,12 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { if (indexingRule.isFulltextEnabled()) { document.add(newFulltextField(name)); } + + if (getDefinition().evaluatePathRestrictions()){ + document.add(newAncestorsField(PathUtils.getParentPath(path))); + document.add(newDepthField(path)); + } + for (Field f : fields) { document.add(f); } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 8bf1e6e..014fb72 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -31,7 +31,6 @@ import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.util.ISO8601; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SerialMergeScheduler; @@ -47,13 +46,13 @@ public class LuceneIndexEditorContext { private static final Logger log = LoggerFactory .getLogger(LuceneIndexEditorContext.class); - private static IndexWriterConfig getIndexWriterConfig(Analyzer analyzer, IndexDefinition definition) { + private static IndexWriterConfig getIndexWriterConfig(IndexDefinition definition) { // FIXME: Hack needed to make Lucene work in an OSGi environment Thread thread = Thread.currentThread(); ClassLoader loader = thread.getContextClassLoader(); thread.setContextClassLoader(IndexWriterConfig.class.getClassLoader()); try { - IndexWriterConfig config = new IndexWriterConfig(VERSION, analyzer); + IndexWriterConfig config = new IndexWriterConfig(VERSION, definition.getAnalyzer()); config.setMergeScheduler(new SerialMergeScheduler()); if (definition.getCodec() != null) { config.setCodec(definition.getCodec()); @@ -104,10 +103,10 @@ public class LuceneIndexEditorContext { private boolean reindex; - LuceneIndexEditorContext(NodeState root, NodeBuilder definition, Analyzer analyzer, IndexUpdateCallback updateCallback) { + LuceneIndexEditorContext(NodeState root, NodeBuilder definition, IndexUpdateCallback updateCallback) { this.definitionBuilder = definition; this.definition = new IndexDefinition(root, definition); - this.config = getIndexWriterConfig(analyzer, this.definition); + this.config = getIndexWriterConfig(this.definition); this.indexedNodes = 0; this.updateCallback = updateCallback; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java index 9e2f229..0944e34 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java @@ -16,7 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ANALYZER; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; import org.apache.felix.scr.annotations.Component; @@ -28,7 +27,6 @@ import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.spi.commit.Editor; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.lucene.analysis.Analyzer; /** * Service that provides Lucene based {@link IndexEditor}s @@ -41,33 +39,15 @@ import org.apache.lucene.analysis.Analyzer; @Service(IndexEditorProvider.class) public class LuceneIndexEditorProvider implements IndexEditorProvider { - /** - * TODO how to inject this in an OSGi friendly way? - */ - private Analyzer analyzer = ANALYZER; - @Override public Editor getIndexEditor( String type, NodeBuilder definition, NodeState root, IndexUpdateCallback callback) throws CommitFailedException { if (TYPE_LUCENE.equals(type)) { - return new LuceneIndexEditor(root, definition, analyzer, callback); + return new LuceneIndexEditor(root, definition, callback); } return null; } - /** - * sets the default analyzer that will be used at index time - */ - public void setAnalyzer(Analyzer analyzer) { - this.analyzer = analyzer; - } - - // ----- helper builder method - - public LuceneIndexEditorProvider with(Analyzer analyzer) { - this.setAnalyzer(analyzer); - return this; - } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java index 5dd255c..216cfdb 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java @@ -28,7 +28,6 @@ import org.apache.jackrabbit.oak.spi.commit.Observer; import org.apache.jackrabbit.oak.spi.query.QueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.lucene.analysis.Analyzer; import com.google.common.collect.ImmutableList; @@ -41,8 +40,6 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea protected final IndexTracker tracker; - protected volatile Analyzer analyzer = LuceneIndexConstants.ANALYZER; - protected volatile NodeAggregator aggregator = null; public LuceneIndexProvider() { @@ -72,18 +69,11 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea } protected LuceneIndex newLuceneIndex() { - return new LuceneIndex(tracker, analyzer, aggregator); + return new LuceneIndex(tracker, aggregator); } protected LucenePropertyIndex newLucenePropertyIndex() { - return new LucenePropertyIndex(tracker, analyzer); - } - - /** - * sets the default analyzer that will be used at query time - */ - public void setAnalyzer(Analyzer analyzer) { - this.analyzer = analyzer; + return new LucenePropertyIndex(tracker); } /** @@ -95,11 +85,6 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea // ----- helper builder method - public LuceneIndexProvider with(Analyzer analyzer) { - this.setAnalyzer(analyzer); - return this; - } - public LuceneIndexProvider with(NodeAggregator analyzer) { this.setAggregator(analyzer); return this; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index 3b6bab4..5ff90e8 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -44,7 +44,6 @@ import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardExecutor; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.util.InfoStream; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceRegistration; @@ -66,20 +65,12 @@ public class LuceneIndexProviderService { private final Logger log = LoggerFactory.getLogger(getClass()); - private final Analyzer defaultAnalyzer = LuceneIndexConstants.ANALYZER; - @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, policyOption = ReferencePolicyOption.GREEDY, policy = ReferencePolicy.DYNAMIC ) private NodeAggregator nodeAggregator; - @Reference(cardinality = ReferenceCardinality.OPTIONAL_UNARY, - policyOption = ReferencePolicyOption.GREEDY, - policy = ReferencePolicy.DYNAMIC - ) - protected Analyzer analyzer; - @Property( boolValue = false, label = "Enable Debug Logging", @@ -156,9 +147,6 @@ public class LuceneIndexProviderService { } indexProvider.setAggregator(nodeAggregator); - - Analyzer analyzer = this.analyzer != null ? this.analyzer : defaultAnalyzer; - indexProvider.setAnalyzer(analyzer); } private void initializeLogging(Map config) { @@ -211,14 +199,4 @@ public class LuceneIndexProviderService { this.nodeAggregator = null; initialize(); } - - protected void bindAnalyzer(Analyzer analyzer) { - this.analyzer = analyzer; - initialize(); - } - - protected void unbindAnalyzer(Analyzer analyzer) { - this.analyzer = null; - initialize(); - } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index 1772d43..36f4af5 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -102,6 +102,7 @@ import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; +import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newFulltextTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; import static org.apache.jackrabbit.oak.query.QueryImpl.JCR_PATH; @@ -165,12 +166,8 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati protected final IndexTracker tracker; - private final Analyzer analyzer; - - public LucenePropertyIndex( - IndexTracker tracker, Analyzer analyzer) { + public LucenePropertyIndex(IndexTracker tracker) { this.tracker = tracker; - this.analyzer = analyzer; } @Override @@ -225,7 +222,7 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati .append("(") .append(path) .append(") "); - sb.append(getQuery(plan, null, analyzer)); + sb.append(getQuery(plan, null)); if(plan.getSortOrder() != null && !plan.getSortOrder().isEmpty()){ sb.append(" ordering:").append(plan.getSortOrder()); } @@ -299,7 +296,7 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati checkState(indexNode != null); try { IndexSearcher searcher = indexNode.getSearcher(); - Query query = getQuery(plan, searcher.getIndexReader(), analyzer); + Query query = getQuery(plan, searcher.getIndexReader()); TopDocs docs; long time = System.currentTimeMillis(); if (lastDoc != null) { @@ -416,12 +413,13 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati * @param defn nodestate that contains the index definition * @return the Lucene query */ - private static Query getQuery(IndexPlan plan, IndexReader reader, Analyzer analyzer) { + private static Query getQuery(IndexPlan plan, IndexReader reader) { List qs = new ArrayList(); Filter filter = plan.getFilter(); FullTextExpression ft = filter.getFullTextConstraint(); PlanResult planResult = pr(plan); IndexDefinition defn = planResult.indexDefinition; + Analyzer analyzer = defn.getAnalyzer(); if (ft == null) { // there might be no full-text constraint // when using the LowCostLuceneIndexProvider @@ -503,26 +501,22 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati addNodeTypeConstraints(planResult.indexingRule, qs, filter); } - String path = filter.getPath(); - //TODO Readjust path based on pathPrefix of the index + String path = getPathRestriction(plan); switch (filter.getPathRestriction()) { case ALL_CHILDREN: if (defn.evaluatePathRestrictions()) { if ("/".equals(path)) { break; } - if (!path.endsWith("/")) { - path += "/"; - } - qs.add(new PrefixQuery(newPathTerm(path))); + qs.add(new TermQuery(newAncestorTerm(path))); } break; case DIRECT_CHILDREN: if (defn.evaluatePathRestrictions()) { - if (!path.endsWith("/")) { - path += "/"; - } - qs.add(new PrefixQuery(newPathTerm(path))); + BooleanQuery bq = new BooleanQuery(); + bq.add(new BooleanClause(new TermQuery(newAncestorTerm(path)), BooleanClause.Occur.MUST)); + bq.add(new BooleanClause(newDepthQuery(path), BooleanClause.Occur.MUST)); + qs.add(bq); } break; case EXACT: @@ -1050,6 +1044,21 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati return tokens; } + private static String getPathRestriction(IndexPlan plan) { + Filter f = plan.getFilter(); + String pathPrefix = plan.getPathPrefix(); + if (pathPrefix.isEmpty()) { + return f.getPath(); + } + String relativePath = PathUtils.relativize(pathPrefix, f.getPath()); + return "/" + relativePath; + } + + private static Query newDepthQuery(String path) { + int depth = PathUtils.getDepth(path) + 1; + return NumericRangeQuery.newIntRange(FieldNames.PATH_DEPTH, depth, depth, true, true); + } + static class LuceneResultRow { final String path; final double score; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/TermFactory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/TermFactory.java index debc146..dd181e1 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/TermFactory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/TermFactory.java @@ -39,10 +39,7 @@ public final class TermFactory { * @return the path term. */ public static Term newPathTerm(String path) { - if (!"/".equals(path) && !path.startsWith("/")) { - path = "/" + path; - } - return new Term(FieldNames.PATH, path); + return new Term(FieldNames.PATH, preparePath(path)); } public static Term newFulltextTerm(String ft, String field) { @@ -52,8 +49,18 @@ public final class TermFactory { return new Term(field, ft); } + public static Term newAncestorTerm(String path){ + return new Term(FieldNames.ANCESTORS, preparePath(path)); + } + public static Term newFulltextTerm(String ft) { return new Term(FieldNames.FULLTEXT, ft); } + private static String preparePath(String path) { + if (!"/".equals(path) && !path.startsWith("/")) { + path = "/" + path; + } + return path; + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TokenizerChain.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TokenizerChain.java new file mode 100644 index 0000000..c49cc86 --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/TokenizerChain.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.util; + +import java.io.Reader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.util.CharFilterFactory; +import org.apache.lucene.analysis.util.TokenFilterFactory; +import org.apache.lucene.analysis.util.TokenizerFactory; + +/** + * An analyzer that uses a tokenizer and a list of token filters to + * create a TokenStream. Taken from org.apache.solr.analysis.TokenizerChain + */ +public final class TokenizerChain extends Analyzer { + private final CharFilterFactory[] charFilters; + private final TokenizerFactory tokenizer; + private final TokenFilterFactory[] filters; + + public TokenizerChain(TokenizerFactory tokenizer) { + this(null, tokenizer, null); + } + + public TokenizerChain(TokenizerFactory tokenizer, TokenFilterFactory[] filters) { + this(null, tokenizer, filters); + } + + public TokenizerChain(CharFilterFactory[] charFilters, TokenizerFactory tokenizer, TokenFilterFactory[] filters) { + this.charFilters = charFilters; + this.tokenizer = tokenizer; + this.filters = filters == null ? new TokenFilterFactory[0] : filters; + } + + @Override + public Reader initReader(String fieldName, Reader reader) { + if (charFilters != null && charFilters.length > 0) { + Reader cs = reader; + for (CharFilterFactory charFilter : charFilters) { + cs = charFilter.create(cs); + } + reader = cs; + } + return reader; + } + + @Override + protected TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer tk = tokenizer.create(reader); + TokenStream ts = tk; + for (TokenFilterFactory filter : filters) { + ts = filter.create(ts); + } + return new TokenStreamComponents(tk, ts); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("TokenizerChain("); + for (CharFilterFactory filter : charFilters) { + sb.append(filter); + sb.append(", "); + } + sb.append(tokenizer); + for (TokenFilterFactory filter : filters) { + sb.append(", "); + sb.append(filter); + } + sb.append(')'); + return sb.toString(); + } + +} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java index 7148331..1a96e77 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java @@ -22,7 +22,6 @@ import java.util.List; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.spi.query.Filter; import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.lucene.analysis.Analyzer; /** * A LuceneIndexProvider that return a LuceneIndex with a really low cost, so @@ -33,13 +32,13 @@ public class LowCostLuceneIndexProvider extends LuceneIndexProvider { @Override protected LuceneIndex newLuceneIndex() { - return new LowCostLuceneIndex(tracker, analyzer, aggregator); + return new LowCostLuceneIndex(tracker, aggregator); } private static class LowCostLuceneIndex extends LuceneIndex { - public LowCostLuceneIndex(IndexTracker tracker, Analyzer analyzer, NodeAggregator aggregator) { - super(tracker, analyzer, aggregator); + public LowCostLuceneIndex(IndexTracker tracker, NodeAggregator aggregator) { + super(tracker, aggregator); } @Override diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java index 087c119..d95d317 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorTest.java @@ -32,7 +32,6 @@ import org.apache.jackrabbit.oak.spi.commit.EditorHook; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.test.ISO8601; -import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; @@ -64,11 +63,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; public class LuceneIndexEditorTest { - private static final Analyzer analyzer = LuceneIndexConstants.ANALYZER; - private static final EditorHook HOOK = new EditorHook( new IndexUpdateProvider( - new LuceneIndexEditorProvider().with(analyzer))); + new LuceneIndexEditorProvider())); private NodeState root = INITIAL_CONTENT; @@ -83,7 +80,7 @@ public class LuceneIndexEditorTest { NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); NodeBuilder idxnb = newLuceneIndexDefinitionV2(index, "lucene", of(TYPENAME_STRING)); - + IndexDefinition defn = new IndexDefinition(root, idxnb.getNodeState()); NodeState before = builder.getNodeState(); builder.child("test").setProperty("foo", "fox is jumping"); builder.child("test").setProperty("price", 100); @@ -93,7 +90,7 @@ public class LuceneIndexEditorTest { tracker.update(indexed); //system fields starts with ':' so need to be escaped - assertEquals("/test", query(escape(FieldNames.createAnalyzedFieldName("foo"))+":fox")); + assertEquals("/test", query(escape(FieldNames.createAnalyzedFieldName("foo"))+":fox", defn)); assertNull("Non string properties not indexed by default", getPath(NumericRangeQuery.newLongRange("price", 100L, 100L, true, true))); } @@ -109,7 +106,7 @@ public class LuceneIndexEditorTest { of(TYPENAME_STRING)); nb.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false); nb.setProperty(createProperty(INCLUDE_PROPERTY_NAMES, of("foo", "price", "weight", "bool", "creationTime"), STRINGS)); - + IndexDefinition defn = new IndexDefinition(root, nb.getNodeState()); NodeState before = builder.getNodeState(); builder.child("test").setProperty("foo", "fox is jumping"); builder.child("test").setProperty("bar", "kite is flying"); @@ -123,7 +120,7 @@ public class LuceneIndexEditorTest { NodeState indexed = HOOK.processCommit(before, after, CommitInfo.EMPTY); tracker.update(indexed); - assertNull("Fulltext search should not work", query("foo:fox")); + assertNull("Fulltext search should not work", query("foo:fox",defn)); assertEquals("/test", getPath(new TermQuery(new Term("foo", "fox is jumping")))); assertNull("bar must NOT be indexed", getPath(new TermQuery(new Term("bar", "kite is flying")))); @@ -340,8 +337,8 @@ public class LuceneIndexEditorTest { indexNode = null; } - private String query(String query) throws IOException, ParseException { - QueryParser queryParser = new QueryParser(VERSION, "", analyzer); + private String query(String query, IndexDefinition defn) throws IOException, ParseException { + QueryParser queryParser = new QueryParser(VERSION, "", defn.getAnalyzer()); return getPath(queryParser.parse(query)); } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java index fda4258..6263008 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java @@ -17,6 +17,7 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import java.io.File; +import java.util.Collections; import java.util.List; import java.util.Set; @@ -26,6 +27,7 @@ import javax.jcr.PropertyType; import static com.google.common.collect.ImmutableList.copyOf; import static com.google.common.collect.Iterators.transform; +import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Sets.newHashSet; import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; import static junit.framework.Assert.assertEquals; @@ -68,6 +70,7 @@ import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.lucene.analysis.Analyzer; import org.junit.After; +import org.junit.Assert; import org.junit.Test; import com.google.common.collect.ImmutableList; @@ -75,11 +78,9 @@ import com.google.common.collect.ImmutableSet; public class LuceneIndexTest { - private static final Analyzer analyzer = LuceneIndexConstants.ANALYZER; - private static final EditorHook HOOK = new EditorHook( new IndexUpdateProvider( - new LuceneIndexEditorProvider().with(analyzer))); + new LuceneIndexEditorProvider())); private NodeState root = INITIAL_CONTENT; @@ -100,7 +101,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, analyzer); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -130,7 +131,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, analyzer); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); FilterImpl filter = createFilter(NT_BASE); filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); @@ -163,7 +164,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, analyzer); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -202,7 +203,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, analyzer); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -216,16 +217,53 @@ public class LuceneIndexTest { assertFalse(cursor.hasNext()); } - private FilterImpl createFilter(String nodeTypeName) { - NodeState system = root.getChildNode(JCR_SYSTEM); - NodeState types = system.getChildNode(JCR_NODE_TYPES); - NodeState type = types.getChildNode(nodeTypeName); - SelectorImpl selector = new SelectorImpl(type, nodeTypeName); - return new FilterImpl(selector, "SELECT * FROM [" + nodeTypeName + "]", new QueryEngineSettings()); + @Test + public void testPathRestrictions() throws Exception { + NodeBuilder idx = newLucenePropertyIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), + "lucene", ImmutableSet.of("foo"), null); + idx.setProperty(LuceneIndexConstants.EVALUATE_PATH_RESTRICTION, true); + + NodeState before = builder.getNodeState(); + builder.setProperty("foo", "bar"); + builder.child("a").setProperty("foo", "bar"); + builder.child("a1").setProperty("foo", "bar"); + builder.child("a").child("b").setProperty("foo", "bar"); + builder.child("a").child("b").child("c").setProperty("foo", "bar"); + + NodeState after = builder.getNodeState(); + + NodeState indexed = HOOK.processCommit(before, after,CommitInfo.EMPTY); + + IndexTracker tracker = new IndexTracker(); + tracker.update(indexed); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); + + FilterImpl filter = createTestFilter(); + filter.restrictPath("/", Filter.PathRestriction.EXACT); + assertFilter(filter, queryIndex, indexed, ImmutableList.of("/")); + + filter = createTestFilter(); + filter.restrictPath("/", Filter.PathRestriction.DIRECT_CHILDREN); + assertFilter(filter, queryIndex, indexed, ImmutableList.of("/a", "/a1")); + + filter = createTestFilter(); + filter.restrictPath("/a", Filter.PathRestriction.DIRECT_CHILDREN); + assertFilter(filter, queryIndex, indexed, ImmutableList.of("/a/b")); + + filter = createTestFilter(); + filter.restrictPath("/a", Filter.PathRestriction.ALL_CHILDREN); + assertFilter(filter, queryIndex, indexed, ImmutableList.of("/a/b", "/a/b/c")); + } + + private FilterImpl createTestFilter(){ + FilterImpl filter = createFilter(NT_BASE); + filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); + return filter; } @Test public void testTokens() { + Analyzer analyzer = LuceneIndexConstants.ANALYZER; assertEquals(ImmutableList.of("parent", "child"), LuceneIndex.tokenize("/parent/child", analyzer)); assertEquals(ImmutableList.of("p1234", "p5678"), @@ -367,9 +405,16 @@ public class LuceneIndexTest { } } + private FilterImpl createFilter(String nodeTypeName) { + NodeState system = root.getChildNode(JCR_SYSTEM); + NodeState types = system.getChildNode(JCR_NODE_TYPES); + NodeState type = types.getChildNode(nodeTypeName); + SelectorImpl selector = new SelectorImpl(type, nodeTypeName); + return new FilterImpl(selector, "SELECT * FROM [" + nodeTypeName + "]", new QueryEngineSettings()); + } private void assertQuery(IndexTracker tracker, NodeState indexed, String key, String value){ - AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, analyzer); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty(key, Operator.EQUAL, @@ -381,6 +426,23 @@ public class LuceneIndexTest { assertFalse(cursor.hasNext()); } + private static List assertFilter(Filter filter, AdvancedQueryIndex queryIndex, + NodeState indexed, List expected) { + List plans = queryIndex.getPlans(filter, null, indexed); + Cursor cursor = queryIndex.query(plans.get(0), indexed); + + List paths = newArrayList(); + while (cursor.hasNext()) { + paths.add(cursor.next().getPath()); + } + Collections.sort(paths); + for (String p : expected) { + assertTrue("Expected path " + p + " not found", paths.contains(p)); + } + assertEquals("Result set size is different", expected.size(), paths.size()); + return paths; + } + private String getIndexDir(){ File dir = new File("target", "indexdir"+System.nanoTime()); dirs.add(dir); diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index a011799..4eb9732 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -422,6 +422,7 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { public void indexDefinitionBelowRoot() throws Exception { Tree parent = root.getTree("/").addChild("test"); Tree idx = createIndex(parent, "test1", of("propa", "propb")); + idx.setProperty(LuceneIndexConstants.EVALUATE_PATH_RESTRICTION, true); idx.addChild(PROP_NODE).addChild("propa"); root.commit(); @@ -432,7 +433,21 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { assertQuery("select [jcr:path] from [nt:base] as s where ISDESCENDANTNODE(s, '/test') and propa = 'a'", asList("/test/test2/a")); } + @Test + public void indexDefinitionBelowRoot2() throws Exception { + Tree parent = root.getTree("/").addChild("test"); + Tree idx = createIndex(parent, "test1", of("propa", "propb")); + idx.setProperty(LuceneIndexConstants.EVALUATE_PATH_RESTRICTION, true); + idx.addChild(PROP_NODE).addChild("propa"); + root.commit(); + Tree test = parent.addChild("test2").addChild("test3"); + test.addChild("a").setProperty("propa", "a"); + root.commit(); + + assertQuery("select [jcr:path] from [nt:base] as s where ISDESCENDANTNODE(s, '/test/test2') and propa = 'a'", + asList("/test/test2/test3/a")); + } @Test public void sortQueriesWithLong() throws Exception { Tree idx = createIndex("test1", of("foo", "bar"));