### Eclipse Workspace Patch 1.0 #P oak-lucene Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (revision 1759821) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java (working copy) @@ -35,6 +35,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.FunctionIndexParser; import org.apache.jackrabbit.oak.query.QueryImpl; import org.apache.jackrabbit.oak.query.fulltext.FullTextContains; import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression; @@ -146,6 +147,16 @@ List indexedProps = newArrayListWithCapacity(filter.getPropertyRestrictions().size()); + for (PropertyDefinition functionIndex : indexingRule.getFunctionRestrictions()) { + for (PropertyRestriction pr : filter.getPropertyRestrictions()) { + String name = functionIndex.function; + String f = FunctionIndexParser.convertToPolishNotation(name); + if (pr.propertyName.equals(f)) { + indexedProps.add(f); + result.propDefns.put(f, functionIndex); + } + } + } //Optimization - Go further only if any of the property is configured //for property index List facetFields = new LinkedList(); @@ -156,7 +167,7 @@ continue; } if (name.startsWith(QueryConstants.FUNCTION_RESTRICTION_PREFIX)) { - // TODO support function-based indexes + // function-based indexes were handled before continue; } if (QueryImpl.REP_FACET.equals(pr.propertyName)) { @@ -504,6 +515,15 @@ // Supports jcr:score descending natively orderEntries.add(IndexDefinition.NATIVE_SORT_ORDER); } + for (PropertyDefinition functionIndex : rule.getFunctionRestrictions()) { + String name = functionIndex.function; + String f = FunctionIndexParser.convertToPolishNotation(name); + if (o.getPropertyName().equals(f)) { + // Lucene can manage any order desc/asc + orderEntries.add(o); + result.sortedProperties.add(functionIndex); + } + } } //TODO Should we return order entries only when all order clauses are satisfied @@ -549,6 +569,14 @@ //Relative parent properties where [../foo1] is not null return true; } + boolean failTestOnMissingFunctionIndex = true; + if (failTestOnMissingFunctionIndex) { + // this means even just function restrictions fail the test + // (for example "where upper(name) = 'X'", + // if a matching function-based index is missing + return false; + } + // the following would ensure the test doesn't fail in that case: for (PropertyRestriction r : filter.getPropertyRestrictions()) { if (!r.propertyName.startsWith(QueryConstants.FUNCTION_RESTRICTION_PREFIX)) { // not a function restriction Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (revision 1759748) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (working copy) @@ -75,7 +75,6 @@ import static com.google.common.collect.Sets.newHashSet; import static org.apache.jackrabbit.JcrConstants.JCR_SCORE; import static org.apache.jackrabbit.JcrConstants.NT_BASE; -import static org.apache.jackrabbit.oak.api.Type.BOOLEAN; import static org.apache.jackrabbit.oak.api.Type.NAMES; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES; @@ -714,6 +713,7 @@ private final Map propConfigs; private final List namePatterns; private final List nullCheckEnabledProperties; + private final List functionRestrictions; private final List notNullCheckEnabledProperties; private final List nodeScopeAnalyzedProps; private final boolean indexesAllNodesOfMatchingType; @@ -739,17 +739,19 @@ List namePatterns = newArrayList(); List nonExistentProperties = newArrayList(); + List functionRestrictions = newArrayList(); List existentProperties = newArrayList(); List nodeScopeAnalyzedProps = newArrayList(); List propIncludes = newArrayList(); this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, - existentProperties, nodeScopeAnalyzedProps); + existentProperties, nodeScopeAnalyzedProps, functionRestrictions); this.propAggregate = new Aggregate(nodeTypeName, propIncludes); this.aggregate = combine(propAggregate, nodeTypeName); this.namePatterns = ImmutableList.copyOf(namePatterns); this.nodeScopeAnalyzedProps = ImmutableList.copyOf(nodeScopeAnalyzedProps); this.nullCheckEnabledProperties = ImmutableList.copyOf(nonExistentProperties); + this.functionRestrictions = ImmutableList.copyOf(functionRestrictions); this.notNullCheckEnabledProperties = ImmutableList.copyOf(existentProperties); this.fulltextEnabled = aggregate.hasNodeAggregates() || hasAnyFullTextEnabledProperty(); this.nodeFullTextIndexed = aggregate.hasNodeAggregates() || anyNodeScopeIndexedProperty(); @@ -778,6 +780,7 @@ this.propAggregate = original.propAggregate; this.nullCheckEnabledProperties = original.nullCheckEnabledProperties; this.notNullCheckEnabledProperties = original.notNullCheckEnabledProperties; + this.functionRestrictions = original.functionRestrictions; this.nodeScopeAnalyzedProps = original.nodeScopeAnalyzedProps; this.aggregate = combine(propAggregate, nodeTypeName); this.fulltextEnabled = aggregate.hasNodeAggregates() || original.fulltextEnabled; @@ -818,6 +821,10 @@ public List getNullCheckEnabledProperties() { return nullCheckEnabledProperties; } + + public List getFunctionRestrictions() { + return functionRestrictions; + } public List getNotNullCheckEnabledProperties() { return notNullCheckEnabledProperties; @@ -941,11 +948,13 @@ return JcrConstants.NT_BASE.equals(baseNodeType); } - private Map collectPropConfigs(NodeState config, List patterns, + private Map collectPropConfigs(NodeState config, + List patterns, List propAggregate, List nonExistentProperties, List existentProperties, - List nodeScopeAnalyzedProps) { + List nodeScopeAnalyzedProps, + List functionRestrictions) { Map propDefns = newHashMap(); NodeState propNode = config.getChildNode(LuceneIndexConstants.PROP_NODE); @@ -965,6 +974,11 @@ NodeState propDefnNode = propNode.getChildNode(propName); if (propDefnNode.exists() && !propDefns.containsKey(propName)) { PropertyDefinition pd = new PropertyDefinition(this, propName, propDefnNode); + if (pd.function != null) { + functionRestrictions.add(pd); + // a function index has no other options + continue; + } if(pd.isRegexp){ patterns.add(new NamePattern(pd.name, pd)); } else { Index: src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (revision 1759748) +++ src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java (working copy) @@ -100,6 +100,14 @@ NodeBuilder props = ntBase.child(LuceneIndexConstants.PROP_NODE); props.setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME); + // Enable function-based indexes: upper+lower(name+localname+prop1) + functionBasedIndex(props, "upper(name())"); + functionBasedIndex(props, "lower(name())"); + functionBasedIndex(props, "upper(localname())"); + functionBasedIndex(props, "lower(localname())"); + functionBasedIndex(props, "upper([prop1])"); + functionBasedIndex(props, "lower([prop1])"); + enableFulltextIndex(props.child("allProps")); } } @@ -115,5 +123,12 @@ .setProperty(LuceneIndexConstants.PROP_NAME, LuceneIndexConstants.REGEX_ALL_PROPS) .setProperty(LuceneIndexConstants.PROP_IS_REGEX, true); } + + private static void functionBasedIndex(NodeBuilder props, String function) { + props.child(function). + setProperty(JCR_PRIMARYTYPE, "nt:unstructured", NAME). + setProperty(LuceneIndexConstants.PROP_FUNCTION, function); + } + } } Index: src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/FunctionIndexTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/FunctionIndexTest.java (revision 0) +++ src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/FunctionIndexTest.java (working copy) @@ -0,0 +1,306 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import static org.apache.jackrabbit.oak.api.QueryEngine.NO_BINDINGS; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +import java.text.ParseException; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import javax.jcr.PropertyType; + +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.Result; +import org.apache.jackrabbit.oak.api.ResultRow; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; +import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent; +import org.apache.jackrabbit.oak.query.AbstractQueryTest; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.junit.Test; + +import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; + +public class FunctionIndexTest extends AbstractQueryTest { + + private LuceneIndexEditorProvider editorProvider; + + private NodeStore nodeStore; + + @Override + protected ContentRepository createRepository() { + editorProvider = new LuceneIndexEditorProvider(); + LuceneIndexProvider provider = new LuceneIndexProvider(); + nodeStore = new MemoryNodeStore(); + return new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with((QueryIndexProvider) provider) + .with((Observer) provider) + .with(editorProvider) + .with(new PropertyIndexEditorProvider()) + .with(new NodeTypeIndexProvider()) + .createContentRepository(); + } + + @Test + public void noIndexTest() throws Exception { + Tree test = root.getTree("/").addChild("test"); + for (int idx = 0; idx < 3; idx++) { + Tree low = test.addChild("" + (char) ('a' + idx)); + low.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + Tree up = test.addChild("" + (char) ('A' + idx)); + up.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + } + root.commit(); + + String query = "select [jcr:path] from [nt:base] where lower(localname()) = 'b'"; + assertThat(explain(query), containsString("traverse")); + assertQuery(query, Lists.newArrayList("/test/b", "/test/B")); + + String queryXPath = "/jcr:root/test//*[fn:lower-case(fn:local-name()) = 'b']"; + assertThat(explainXpath(queryXPath), containsString("traverse")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/b", "/test/B")); + + queryXPath = "/jcr:root/test//*[fn:lower-case(fn:local-name()) > 'b']"; + assertThat(explainXpath(queryXPath), containsString("traverse")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/c", "/test/C")); + + query = "select [jcr:path] from [nt:base] where lower(localname()) = 'B'"; + assertThat(explain(query), containsString("traverse")); + assertQuery(query, Lists.newArrayList()); + } + + @Test + public void lowerCaseLocalName() throws Exception { + Tree luceneIndex = createIndex("lowerLocalName", Collections.emptySet()); + luceneIndex.setProperty("excludedPaths", + Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS); + Tree func = luceneIndex.addChild(LuceneIndexConstants.INDEX_RULES) + .addChild("nt:base") + .addChild(LuceneIndexConstants.PROP_NODE) + .addChild("lowerLocalName"); + func.setProperty(LuceneIndexConstants.PROP_FUNCTION, "lower(localname())"); + + Tree test = root.getTree("/").addChild("test"); + for (int idx = 0; idx < 3; idx++) { + Tree low = test.addChild("" + (char) ('a' + idx)); + low.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + Tree up = test.addChild("" + (char) ('A' + idx)); + up.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + } + root.commit(); + + String query = "select [jcr:path] from [nt:base] where lower(localname()) = 'b'"; + assertThat(explain(query), containsString("lucene:lowerLocalName")); + assertQuery(query, Lists.newArrayList("/test/b", "/test/B")); + + String queryXPath = "/jcr:root//*[fn:lower-case(fn:local-name()) = 'b']"; + assertThat(explainXpath(queryXPath), containsString("lucene:lowerLocalName")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/b", "/test/B")); + + queryXPath = "/jcr:root//*[fn:lower-case(fn:local-name()) > 'b']"; + assertThat(explainXpath(queryXPath), containsString("lucene:lowerLocalName")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/c", "/test/C", "/test")); + + query = "select [jcr:path] from [nt:base] where lower(localname()) = 'B'"; + assertThat(explain(query), containsString("lucene:lowerLocalName")); + assertQuery(query, Lists.newArrayList()); + } + + @Test + public void lengthName() throws Exception { + Tree luceneIndex = createIndex("lengthName", Collections.emptySet()); + luceneIndex.setProperty("excludedPaths", + Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS); + Tree func = luceneIndex.addChild(LuceneIndexConstants.INDEX_RULES) + .addChild("nt:base") + .addChild(LuceneIndexConstants.PROP_NODE) + .addChild("lengthName"); + func.setProperty(LuceneIndexConstants.PROP_ORDERED, true); + func.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG); + func.setProperty(LuceneIndexConstants.PROP_FUNCTION, "fn:string-length(fn:name())"); + + Tree test = root.getTree("/").addChild("test"); + for (int idx = 1; idx < 1000; idx *= 10) { + Tree testNode = test.addChild("test" + idx); + testNode.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + } + root.commit(); + + String query = "select [jcr:path] from [nt:base] where length(name()) = 6"; + assertThat(explain(query), containsString("lucene:lengthName")); + assertQuery(query, Lists.newArrayList("/test/test10")); + + String queryXPath = "/jcr:root//*[fn:string-length(fn:name()) = 7]"; + assertThat(explainXpath(queryXPath), containsString("lucene:lengthName")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/test100")); + + queryXPath = "/jcr:root//* order by fn:string-length(fn:name())"; + assertThat(explainXpath(queryXPath), containsString("lucene:lengthName")); + assertQuery(queryXPath, "xpath", Lists.newArrayList( + "/test", "/test/test1", "/test/test10", "/test/test100")); + } + + @Test + public void length() throws Exception { + Tree luceneIndex = createIndex("length", Collections.emptySet()); + luceneIndex.setProperty("excludedPaths", + Lists.newArrayList("/jcr:system", "/oak:index"), Type.STRINGS); + Tree func = luceneIndex.addChild(LuceneIndexConstants.INDEX_RULES) + .addChild("nt:base") + .addChild(LuceneIndexConstants.PROP_NODE) + .addChild("lengthName"); + func.setProperty(LuceneIndexConstants.PROP_FUNCTION, "fn:string-length(@value)"); + + Tree test = root.getTree("/").addChild("test"); + for (int idx = 1; idx <= 1000; idx *= 10) { + Tree testNode = test.addChild("test" + idx); + testNode.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + testNode.setProperty("value", new byte[idx]); + } + root.commit(); + + String query = "select [jcr:path] from [nt:base] where length([value]) = 100"; + assertThat(explain(query), containsString("lucene:length")); + assertQuery(query, Lists.newArrayList("/test/test100")); + + String queryXPath = "/jcr:root//*[fn:string-length(@value) = 10]"; + assertThat(explainXpath(queryXPath), containsString("lucene:length")); + assertQuery(queryXPath, "xpath", Lists.newArrayList("/test/test10")); + } + + @Test + public void upperCase() throws Exception { + Tree luceneIndex = createIndex("upper", Collections.emptySet()); + Tree func = luceneIndex.addChild(LuceneIndexConstants.INDEX_RULES) + .addChild("nt:base") + .addChild(LuceneIndexConstants.PROP_NODE) + .addChild("upperName"); + func.setProperty(LuceneIndexConstants.PROP_FUNCTION, "fn:upper-case(@name)"); + + Tree test = root.getTree("/").addChild("test"); + test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + + List paths = Lists.newArrayList(); + for (int idx = 0; idx < 15; idx++) { + Tree a = test.addChild("n"+idx); + a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + a.setProperty("name", "10% foo"); + paths.add("/test/n" + idx); + } + root.commit(); + + String query = "select [jcr:path] from [nt:unstructured] where upper([name]) = '10% FOO'"; + assertThat(explain(query), containsString("lucene:upper")); + assertQuery(query, paths); + + query = "select [jcr:path] from [nt:unstructured] where upper([name]) like '10\\% FOO'"; + assertThat(explain(query), containsString("lucene:upper")); + assertQuery(query, paths); + + } + + @Test + public void upperCaseOrdering() throws Exception { + // TODO + } + + @Test + public void upperCaseRelative() throws Exception { + Tree luceneIndex = createIndex("upper", Collections.emptySet()); + Tree func = luceneIndex.addChild(LuceneIndexConstants.INDEX_RULES) + .addChild("nt:base") + .addChild(LuceneIndexConstants.PROP_NODE) + .addChild("upperName"); + func.setProperty(LuceneIndexConstants.PROP_FUNCTION, "upper([data/name])"); + + Tree test = root.getTree("/").addChild("test"); + test.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + + List paths = Lists.newArrayList(); + for (int idx = 0; idx < 15; idx++) { + Tree a = test.addChild("n"+idx); + a.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + Tree b = a.addChild("data"); + b.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME); + b.setProperty("name", "foo"); + paths.add("/test/n" + idx); + } + root.commit(); + + String query = "select [jcr:path] from [nt:unstructured] where upper([data/name]) = 'FOO'"; + assertThat(explain(query), containsString("lucene:upper")); + assertQuery(query, paths); + + String queryXPath = "/jcr:root//element(*, nt:unstructured)[fn:upper-case(data/@name) = 'FOO']"; + assertThat(explainXpath(queryXPath), containsString("lucene:upper")); + assertQuery(queryXPath, "xpath", paths); + + } + + protected String explain(String query){ + String explain = "explain " + query; + return executeQuery(explain, "JCR-SQL2").get(0); + } + + protected String explainXpath(String query) throws ParseException { + String explain = "explain " + query; + Result result = executeQuery(explain, "xpath", NO_BINDINGS); + ResultRow row = Iterables.getOnlyElement(result.getRows()); + String plan = row.getValue("plan").getValue(Type.STRING); + return plan; + } + + protected Tree createIndex(String name, Set propNames) { + Tree index = root.getTree("/"); + return createIndex(index, name, propNames); + } + + static Tree createIndex(Tree index, String name, Set propNames) { + Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name); + def.setProperty(JcrConstants.JCR_PRIMARYTYPE, + INDEX_DEFINITIONS_NODE_TYPE, Type.NAME); + def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE); + def.setProperty(REINDEX_PROPERTY_NAME, true); + def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false); + def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES, propNames, Type.STRINGS)); + def.setProperty(LuceneIndexConstants.SAVE_DIR_LISTING, true); + return index.getChild(INDEX_DEFINITIONS_NAME).getChild(name); + } + +} Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (revision 1759748) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java (working copy) @@ -40,8 +40,10 @@ import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText; import org.apache.jackrabbit.oak.plugins.index.fulltext.ExtractedText.ExtractionResult; import org.apache.jackrabbit.oak.plugins.index.lucene.Aggregate.Matcher; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.FunctionIndexParser; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState; +import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; import org.apache.jackrabbit.oak.plugins.memory.StringPropertyState; import org.apache.jackrabbit.oak.plugins.tree.TreeFactory; import org.apache.jackrabbit.oak.spi.commit.Editor; @@ -334,6 +336,7 @@ dirty |= indexAggregates(path, fields, state); dirty |= indexNullCheckEnabledProps(path, fields, state); + dirty |= indexFunctionRestrictions(path, fields, state); dirty |= indexNotNullCheckEnabledProps(path, fields, state); dirty |= augmentCustomFields(path, fields, state); @@ -669,6 +672,101 @@ return fieldAdded; } + private boolean indexFunctionRestrictions(String path, List fields, NodeState state) { + boolean fieldAdded = false; + for (PropertyDefinition pd : indexingRule.getFunctionRestrictions()) { + String f = FunctionIndexParser.convertToPolishNotation(pd.function); + PropertyState functionValue = calculateValue(path, state, f); + if (functionValue != null) { + if (pd.ordered) { + addTypedOrderedFields(fields, functionValue, f, pd); + } + addTypedFields(fields, functionValue, f); + fieldAdded = true; + } + } + return fieldAdded; + } + + private static PropertyState calculateValue(String path, NodeState state, String function) { + try { + return tryCalculateValue(path, state, function); + } catch (RuntimeException e) { + log.error("Failed to calculate function value for {} at {}", function, path, e); + throw e; + } + } + + private static PropertyState tryCalculateValue(String path, NodeState state, String function) { + String[] list = function.split("\\*"); + if (list.length < 2) { + return null; + } + String functionName = list[1]; + String propertyName = list[2]; + if (propertyName.startsWith("@")) { + propertyName = propertyName.substring(1); + } + if (PathUtils.getDepth(propertyName) != 1) { + for(String n : PathUtils.elements(PathUtils.getParentPath(propertyName))) { + state = state.getChildNode(n); + if (!state.exists()) { + return null; + } + } + propertyName = PathUtils.getName(propertyName); + } + PropertyState ps; + if (":localname".equals(propertyName)) { + ps = PropertyStates.createProperty("value", + getLocalName(PathUtils.getName(path)), Type.STRING); + } else if (":name".equals(propertyName)) { + ps = PropertyStates.createProperty("value", + PathUtils.getName(path), Type.STRING); + } else { + ps = state.getProperty(propertyName); + } + if (ps == null || ps.count() == 0) { + return null; + } + Type type = null; + ArrayList values = new ArrayList(ps.count()); + for (int i = 0; i < ps.count(); i++) { + String s = ps.getValue(Type.STRING, i); + Object x; + if ("lower".equals(functionName)) { + x = s.toLowerCase(); + type = Type.STRING; + } else if ("upper".equals(functionName)) { + x = s.toUpperCase(); + type = Type.STRING; + } else if ("length".equals(functionName)) { + x = (long) s.length(); + type = Type.LONG; + } else { + log.debug("Unknown function {}", function); + return null; + } + values.add(x); + } + PropertyState result; + if (values.size() == 1) { + result = PropertyStates.createProperty( + "value", values.get(0), type); + } else { + type = type.getArrayType(); + result = PropertyStates.createProperty( + "value", values, type); + } + return result; + } + + private static String getLocalName(String name) { + int colon = name.indexOf(':'); + // TODO LOCALNAME: evaluation of local name might not be correct + return colon < 0 ? name : name.substring(colon + 1); + } + private boolean indexIfSinglePropertyRemoved() { boolean dirty = false; for (PropertyState ps : propertiesModified) { Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (revision 1759748) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java (working copy) @@ -90,6 +90,11 @@ * Name of the field that contains the node name */ public static final String NODE_NAME = ":nodeName"; + + /** + * Suffix of the fields that contains function values + */ + public static final String FUNCTION_PREFIX = "function*"; /** * Used to select only the PATH field from the lucene documents Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParser.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParser.java (revision 0) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParser.java (working copy) @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.util; + +import org.apache.jackrabbit.oak.spi.query.QueryConstants; + +/** + * A parser for function-based indexes. It converts the human-readable function + * definition (XPath) to the internal Polish notation. + */ +public class FunctionIndexParser { + + private String remaining; + + private FunctionIndexParser(String function) { + this.remaining = function; + } + + public static String convertToPolishNotation(String function) { + FunctionIndexParser p = new FunctionIndexParser(function); + return QueryConstants.FUNCTION_RESTRICTION_PREFIX + p.parse(); + } + + String parse() { + if (match("fn:local-name()") || match("localname()")) { + return "@:localname"; + } + if (match("fn:name()") || match("name()")) { + return "@:name"; + } + if (match("fn:upper-case(") || match("upper(")) { + return "upper*" + parse() + read(")"); + } + if (match("fn:lower-case(") || match("lower(")) { + return "lower*" + parse() + read(")"); + } + if (match("fn:string-length(") || match("length(")) { + return "length*" + parse() + read(")"); + } + int end = remaining.indexOf(')'); + if (end >= 0) { + remaining = remaining.substring(0, end); + } + if (remaining.startsWith("[")) { + return "@" + remaining.substring(1, remaining.lastIndexOf(']')).replaceAll("]]", "]"); + } + // property name + return "@" + remaining.replaceAll("@", ""); + } + + private String read(String string) { + match(string); + return ""; + } + + private boolean match(String string) { + if (remaining.startsWith(string)) { + remaining = remaining.substring(string.length()); + return true; + } + return false; + } + +} Index: src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java (revision 1759748) +++ src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java (working copy) @@ -65,6 +65,9 @@ TestUtil.enableForFullText(props, LuceneIndexConstants.REGEX_ALL_PROPS, true); TestUtil.enablePropertyIndex(props, "a/name", false); TestUtil.enablePropertyIndex(props, "b/name", false); + TestUtil.enableFunctionIndex(props, "length([name])"); + TestUtil.enableFunctionIndex(props, "lower([name])"); + TestUtil.enableFunctionIndex(props, "upper([name])"); root.commit(); } Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (revision 1759748) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java (working copy) @@ -337,4 +337,9 @@ * Boolean property indicate that property should not be included in aggregation */ String PROP_EXCLUDE_FROM_AGGREGATE = "excludeFromAggregation"; + + /** + * String property: the function to index, for function-based index + */ + String PROP_FUNCTION = "function"; } Index: src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java =================================================================== --- src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (revision 1759748) +++ src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java (working copy) @@ -99,6 +99,11 @@ @CheckForNull final String nonRelativeName; + /** + * For function-based indexes: the function, in Polish notation. + */ + final String function; + public PropertyDefinition(IndexingRule idxDefn, String nodeName, NodeState defn) { this.isRegexp = getOptionalValue(defn, PROP_IS_REGEX, false); this.name = getName(defn, nodeName); @@ -134,6 +139,7 @@ this.nonRelativeName = determineNonRelativeName(); this.ancestors = computeAncestors(name); this.facet = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_FACETS, false); + this.function = getOptionalValue(defn, LuceneIndexConstants.PROP_FUNCTION, null); validate(); } Index: src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java (revision 1759748) +++ src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/TestUtil.java (working copy) @@ -112,6 +112,12 @@ return prop; } + public static Tree enableFunctionIndex(Tree props, String function) { + Tree prop = props.addChild(unique("prop")); + prop.setProperty(LuceneIndexConstants.PROP_FUNCTION, function); + return prop; + } + public static AggregatorBuilder newNodeAggregator(Tree indexDefn){ return new AggregatorBuilder(indexDefn); } Index: src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParserTest.java =================================================================== --- src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParserTest.java (revision 0) +++ src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/FunctionIndexParserTest.java (working copy) @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.util; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class FunctionIndexParserTest { + + @Test + public void xpath() { + checkConvert( + "fn:upper-case(@data)", + "function*upper*@data"); + checkConvert( + "fn:lower-case(test/@data)", + "function*lower*@test/data"); + checkConvert( + "fn:lower-case(fn:name())", + "function*lower*@:name"); + checkConvert( + "fn:lower-case(fn:local-name())", + "function*lower*@:localname"); + checkConvert( + "fn:string-length(test/@data)", + "function*length*@test/data"); + checkConvert( + "fn:string-length(fn:name())", + "function*length*@:name"); + checkConvert( + "fn:lower-case(fn:upper-case(test/@data))", + "function*lower*upper*@test/data"); + } + + @Test + public void sql2() { + checkConvert( + "upper([data])", + "function*upper*@data"); + checkConvert( + "lower([test/data])", + "function*lower*@test/data"); + checkConvert( + "lower(name())", + "function*lower*@:name"); + checkConvert( + "lower(localname())", + "function*lower*@:localname"); + checkConvert( + "length([test/data])", + "function*length*@test/data"); + checkConvert( + "length(name())", + "function*length*@:name"); + checkConvert( + "lower(upper([test/data]))", + "function*lower*upper*@test/data"); + // the ']' character is escaped as ']]' + checkConvert( + "[strange[0]]]", + "function*@strange[0]"); + } + + private static void checkConvert(String function, String expectedPolishNotation) { + String p = FunctionIndexParser.convertToPolishNotation(function); + assertEquals(expectedPolishNotation, p); + } + +}