diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/PathFilter.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/PathFilter.java new file mode 100644 index 0000000..69a12d0 --- /dev/null +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/PathFilter.java @@ -0,0 +1,172 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import javax.annotation.Nonnull; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.Collections.singletonList; +import static org.apache.jackrabbit.oak.commons.PathUtils.isAncestor; + +/** + * Filter which determines whether given path should be included for processing + * or not + */ +public class PathFilter { + private static final Collection INCLUDE_ROOT = singletonList("/"); + /** + * Multi value property name used to determine list of paths to be included + */ + public static final String PROP_INCLUDED_PATHS = "includedPaths"; + + /** + * Multi value property name used to determine list of paths to be excluded + */ + public static final String PROP_EXCLUDED_PATHS = "excludedPaths"; + + public enum Result { + /** + * Include the path for processing + */ + INCLUDE, + /** + * Exclude the path and subtree for processing + */ + EXCLUDE, + /** + * Do not process the path but just perform traversal to + * child nodes. For IndexEditor it means that such nodes + * should not be indexed but editor must traverse down + */ + TRAVERSE + } + + private static final PathFilter ALL = new PathFilter(INCLUDE_ROOT, Collections.emptyList()) { + @Override + public Result doFiler(@Nonnull String path) { + return Result.INCLUDE; + } + }; + + private final String[] includedPaths; + private final String[] excludedPaths; + + /** + * Constructs the predicate based on given definition state. It looks + * for multi value property with names {@link PathFilter#PROP_INCLUDED_PATHS} + * and {@link PathFilter#PROP_EXCLUDED_PATHS}. Both the properties + * are optional. + * + * @param defn nodestate representing the configuration. Generally it would be the nodestate + * representing the index definition + * @return predicate based on the passed definition state + */ + public static PathFilter from(NodeBuilder defn){ + if (!defn.hasProperty(PROP_EXCLUDED_PATHS) && !defn.hasProperty(PROP_INCLUDED_PATHS)){ + return ALL; + } + return new PathFilter(getStrings(defn, PROP_INCLUDED_PATHS, INCLUDE_ROOT), + getStrings(defn, PROP_EXCLUDED_PATHS, Collections.emptyList())); + } + + /** + * Constructs the predicate with given included and excluded paths + * + * If both are empty then all paths would be considered to be included + * + * @param includes list of paths which should not be included + * @param excludes list of p + * aths which should be included + */ + public PathFilter(Iterable includes, Iterable excludes) { + Set includeCopy = newHashSet(includes); + Set excludeCopy = newHashSet(excludes); + optimise(includeCopy, excludeCopy); + checkState(!includeCopy.isEmpty(), "No valid include provided. Includes %s, " + + "Excludes %s", includes, excludes); + this.includedPaths = includeCopy.toArray(new String[includeCopy.size()]); + this.excludedPaths = excludeCopy.toArray(new String[excludeCopy.size()]); + } + + /** + * Determines whether given path is to be included or not + * + * @param path path to check + * @return result indicating if the path needs to be included, excluded or just traversed + */ + public Result doFiler(@Nonnull String path) { + for (String excludedPath : excludedPaths){ + if (excludedPath.equals(path) || isAncestor(excludedPath, path)){ + return Result.EXCLUDE; + } + } + + for (String includedPath : includedPaths){ + if (includedPath.equals(path) || isAncestor(includedPath, path)){ + return Result.INCLUDE; + } + } + + for (String includedPath : includedPaths){ + if (includedPath.startsWith(path)){ + return Result.TRAVERSE; + } + } + + return Result.EXCLUDE; + } + + /** + * Removes paths from {@code includePaths} that are completely excluded + * and only retains paths in {@code excludedPaths} that are included. + */ + private static void optimise(Set includePaths, Set excludedPaths) { + Set retain = newHashSet(); + for (String include : includePaths) { + for (String exclude : excludedPaths) { + if (exclude.equals(include) || isAncestor(exclude, include)) { + includePaths.remove(include); + } else if (isAncestor(include, exclude)) { + retain.add(exclude); + } + } + } + excludedPaths.retainAll(retain); + } + + private static Iterable getStrings(NodeBuilder builder, String name, Collection defaultVal) { + PropertyState property = builder.getProperty(name); + if (property != null && property.getType() == Type.STRINGS) { + return property.getValue(Type.STRINGS); + } else { + return defaultVal; + } + } +} diff --git oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/PathFilterTest.java oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/PathFilterTest.java new file mode 100644 index 0000000..5495090 --- /dev/null +++ oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/PathFilterTest.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index; + +import java.util.Collections; + +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.junit.Test; + +import static com.google.common.collect.ImmutableList.of; +import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_EXCLUDED_PATHS; +import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_INCLUDED_PATHS; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class PathFilterTest { + + @Test + public void exclude() throws Exception{ + PathFilter p = new PathFilter(of("/"), of("/etc")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/a")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow")); + } + + @Test + public void include() throws Exception{ + PathFilter p = new PathFilter(of("/content", "/etc"), of("/etc/workflow/instance")); + assertEquals(PathFilter.Result.TRAVERSE, p.doFiler("/")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/var")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/content")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/content/example")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc/workflow")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow/instance")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow/instance/1")); + } + + @Test + public void emptyConfig() throws Exception{ + NodeBuilder root = EMPTY_NODE.builder(); + PathFilter p = PathFilter.from(root); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/a")); + } + + @Test + public void config() throws Exception{ + NodeBuilder root = EMPTY_NODE.builder(); + root.setProperty(createProperty(PROP_INCLUDED_PATHS, of("/etc"), Type.STRINGS)); + root.setProperty(createProperty(PROP_EXCLUDED_PATHS, of("/etc/workflow"), Type.STRINGS)); + PathFilter p = PathFilter.from(root); + assertEquals(PathFilter.Result.TRAVERSE, p.doFiler("/")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc/a")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow/1")); + } + + @Test + public void configOnlyExclude() throws Exception{ + NodeBuilder root = EMPTY_NODE.builder(); + root.setProperty(createProperty(PROP_EXCLUDED_PATHS, of("/etc/workflow"), Type.STRINGS)); + PathFilter p = PathFilter.from(root); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc")); + assertEquals(PathFilter.Result.INCLUDE, p.doFiler("/etc/a")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow")); + assertEquals(PathFilter.Result.EXCLUDE, p.doFiler("/etc/workflow/1")); + } + + @Test + public void invalid() throws Exception{ + try { + new PathFilter(Collections.emptyList(), of("/etc")); + fail(); + } catch (IllegalStateException ignore){ + + } + + try { + new PathFilter(of("/etc/workflow"), of("/etc")); + fail(); + } catch (IllegalStateException ignore){ + + } + + try { + new PathFilter(Collections.emptyList(), Collections.emptyList()); + fail(); + } catch (IllegalStateException ignore){ + + } + } +} diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 721b6f8..5cb7680 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -48,6 +48,7 @@ import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexEditor; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.PathFilter; import org.apache.jackrabbit.oak.plugins.index.lucene.Aggregate.Matcher; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; import org.apache.jackrabbit.oak.plugins.tree.TreeFactory; @@ -112,6 +113,8 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { private final MatcherState matcherState; + private final PathFilter.Result pathFilterResult; + LuceneIndexEditor(NodeState root, NodeBuilder definition, IndexUpdateCallback updateCallback) throws CommitFailedException { this.parent = null; @@ -122,10 +125,12 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { this.root = root; this.isDeleted = false; this.matcherState = MatcherState.NONE; + this.pathFilterResult = context.getPathFilter().doFiler(getPath()); } private LuceneIndexEditor(LuceneIndexEditor parent, String name, MatcherState matcherState, + PathFilter.Result pathFilterResult, boolean isDeleted) { this.parent = parent; this.name = name; @@ -134,6 +139,7 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { this.root = parent.root; this.isDeleted = isDeleted; this.matcherState = matcherState; + this.pathFilterResult = pathFilterResult; } public String getPath() { @@ -159,12 +165,17 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { beforeTree = parent.beforeTree.getChild(name); } - //For traversal in deleted sub tree before state has to be used - Tree current = afterTree.exists() ? afterTree : beforeTree; - indexingRule = getDefinition().getApplicableIndexingRule(current); + //Only check for indexing if the result is include. + //In case like TRAVERSE nothing needs to be indexed for those + //path + if (pathFilterResult == PathFilter.Result.INCLUDE) { + //For traversal in deleted sub tree before state has to be used + Tree current = afterTree.exists() ? afterTree : beforeTree; + indexingRule = getDefinition().getApplicableIndexingRule(current); - if (indexingRule != null) { - currentMatchers = indexingRule.getAggregate().createMatchers(this); + if (indexingRule != null) { + currentMatchers = indexingRule.getAggregate().createMatchers(this); + } } } @@ -218,18 +229,30 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { @Override public Editor childNodeAdded(String name, NodeState after) { - return new LuceneIndexEditor(this, name, getMatcherState(name, after), false); + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult != PathFilter.Result.EXCLUDE) { + return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false); + } + return null; } @Override public Editor childNodeChanged( String name, NodeState before, NodeState after) { - return new LuceneIndexEditor(this, name, getMatcherState(name, after), false); + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult != PathFilter.Result.EXCLUDE) { + return new LuceneIndexEditor(this, name, getMatcherState(name, after), filterResult, false); + } + return null; } @Override public Editor childNodeDeleted(String name, NodeState before) throws CommitFailedException { + PathFilter.Result filterResult = getPathFilterResult(name); + if (filterResult == PathFilter.Result.EXCLUDE) { + return null; + } if (!isDeleted) { // tree deletion is handled on the parent node @@ -249,7 +272,7 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { MatcherState ms = getMatcherState(name, before); if (!ms.isEmpty()){ - return new LuceneIndexEditor(this, name, ms, true); + return new LuceneIndexEditor(this, name, ms, filterResult, true); } return null; // no need to recurse down the removed subtree } @@ -767,6 +790,10 @@ public class LuceneIndexEditor implements IndexEditor, Aggregate.AggregateRoot { return indexingRule != null; } + private PathFilter.Result getPathFilterResult(String childNodeName) { + return context.getPathFilter().doFiler(concat(getPath(), childNodeName)); + } + private boolean isSupportedMediaType(String type) { return context.isSupportedMediaType(type); } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 0fe4382..b51b1c3 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -33,6 +33,7 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.PathFilter; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -118,6 +119,8 @@ public class LuceneIndexEditorContext { private Parser parser; + private final PathFilter pathFilter; + /** * The media types supported by the parser used. */ @@ -125,6 +128,7 @@ public class LuceneIndexEditorContext { LuceneIndexEditorContext(NodeState root, NodeBuilder definition, IndexUpdateCallback updateCallback) { this.definitionBuilder = definition; + this.pathFilter = PathFilter.from(definition); this.definition = new IndexDefinition(root, definition); this.config = getIndexWriterConfig(this.definition); this.indexedNodes = 0; @@ -214,6 +218,10 @@ public class LuceneIndexEditorContext { } } + public PathFilter getPathFilter() { + return pathFilter; + } + public void enableReindexMode(){ reindex = true; IndexFormatVersion version = IndexDefinition.determineVersionForFreshIndex(definitionBuilder); diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java index 57e2297..6b3feaa 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java @@ -67,6 +67,8 @@ import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFIN import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_EXCLUDED_PATHS; +import static org.apache.jackrabbit.oak.plugins.index.PathFilter.PROP_INCLUDED_PATHS; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.ORDERED_PROP_NAMES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_NODE; @@ -364,9 +366,9 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { String explain = explain(q); System.out.println(explain); String luceneQuery = explain.substring(0, explain.indexOf('\n')); - assertEquals("[nt:unstructured] as [content] /* lucene:test1(/oak:index/test1) " + - "+(tags:Products:A tags:Products:A/B) " + - "+(tags:DocTypes:A tags:DocTypes:B tags:DocTypes:C tags:ProblemType:A)", + assertEquals("[nt:unstructured] as [content] /* lucene:test1(/oak:index/test1) " + + "+(tags:Products:A tags:Products:A/B) " + + "+(tags:DocTypes:A tags:DocTypes:B tags:DocTypes:C tags:ProblemType:A)", luceneQuery); } @@ -496,6 +498,50 @@ public class LucenePropertyIndexTest extends AbstractQueryTest { } @Test + public void pathInclude() throws Exception{ + Tree idx = createIndex("test1", of("propa", "propb")); + idx.setProperty(createProperty(PROP_INCLUDED_PATHS, of("/test/a"), Type.STRINGS)); + //Do not provide type information + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + test.addChild("a").setProperty("propa", 10); + test.addChild("a").addChild("b").setProperty("propa", 10); + test.addChild("c").setProperty("propa", 10); + root.commit(); + + assertThat(explain("select [jcr:path] from [nt:base] where [propa] = 10"), containsString("lucene:test1")); + + assertQuery("select [jcr:path] from [nt:base] where [propa] = 10", asList("/test/a", "/test/a/b")); + } + + @Test + public void pathExclude() throws Exception{ + Tree idx = createIndex("test1", of("propa", "propb")); + idx.setProperty(createProperty(PROP_EXCLUDED_PATHS, of("/test/a"), Type.STRINGS)); + //Do not provide type information + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + test.addChild("a").setProperty("propa", 10); + test.addChild("a").addChild("b").setProperty("propa", 10); + test.addChild("c").setProperty("propa", 10); + root.commit(); + + assertThat(explain("select [jcr:path] from [nt:base] where [propa] = 10"), containsString("lucene:test1")); + + assertQuery("select [jcr:path] from [nt:base] where [propa] = 10", asList("/test/c")); + + //Make some change and then check + test = root.getTree("/").getChild("test"); + test.addChild("a").addChild("e").setProperty("propa", 10); + test.addChild("f").setProperty("propa", 10); + root.commit(); + + assertQuery("select [jcr:path] from [nt:base] where [propa] = 10", asList("/test/c", "/test/f")); + } + + @Test public void determinePropTypeFromRestriction() throws Exception{ Tree idx = createIndex("test1", of("propa", "propb")); //Do not provide type information