diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateAdvanceQueryIndex.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateAdvanceQueryIndex.java new file mode 100644 index 0000000..610f35f --- /dev/null +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateAdvanceQueryIndex.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.aggregate; + +import java.util.List; + +import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.QueryIndex; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +public class AggregateAdvanceQueryIndex extends AggregateIndex implements QueryIndex.AdvancedQueryIndex { + private final AdvancedQueryIndex advIndex; + + public AggregateAdvanceQueryIndex(FulltextQueryIndex index){ + super(index); + this.advIndex = (AdvancedQueryIndex) index; + } + + @Override + public List getPlans(Filter filter, List sortOrder, NodeState rootState) { + return advIndex.getPlans(filter, sortOrder, rootState); + } + + @Override + public String getPlanDescription(IndexPlan plan, NodeState root) { + if(!plan.isFulltextIndex()){ + return advIndex.getPlanDescription(plan, root); + } + return super.getPlan(plan.getFilter(), root); + } + + @Override + public Cursor query(IndexPlan plan, NodeState rootState) { + if (!plan.isFulltextIndex()) { + return advIndex.query(plan, rootState); + } + return super.query(plan.getFilter(), rootState); + } +} diff --git oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexProvider.java oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexProvider.java index b73478b..2ee4ed4 100644 --- oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexProvider.java +++ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/aggregate/AggregateIndexProvider.java @@ -26,6 +26,8 @@ import org.apache.jackrabbit.oak.spi.query.QueryIndex.FulltextQueryIndex; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; +import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvancedQueryIndex; + /** * A provider for aggregate indexes. It wraps all full-text query indexes. */ @@ -50,8 +52,14 @@ public class AggregateIndexProvider implements QueryIndexProvider { for (int i = 0; i < list.size(); i++) { QueryIndex index = list.get(i); if (index instanceof FulltextQueryIndex) { - aggregateList - .add(new AggregateIndex((FulltextQueryIndex) index)); + if(index instanceof AdvancedQueryIndex){ + aggregateList + .add(new AggregateAdvanceQueryIndex((FulltextQueryIndex) index)); + + } else { + aggregateList + .add(new AggregateIndex((FulltextQueryIndex) index)); + } } } return aggregateList; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java index 4e55c7f..1171a36 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldFactory.java @@ -16,11 +16,18 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import com.google.common.primitives.Ints; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.util.ISO8601; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.NAME; import static org.apache.lucene.index.FieldInfo.IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; @@ -41,6 +48,13 @@ public final class FieldFactory { private static final FieldType OAK_TYPE_NOT_STORED = new FieldType(); + private static final int[] TYPABLE_TAGS = { + Type.DATE.tag(), + Type.BOOLEAN.tag(), + Type.DOUBLE.tag(), + Type.LONG.tag(), + }; + static { OAK_TYPE.setIndexed(true); OAK_TYPE.setOmitNorms(true); @@ -55,6 +69,8 @@ public final class FieldFactory { OAK_TYPE_NOT_STORED.setIndexOptions(DOCS_AND_FREQS_AND_POSITIONS); OAK_TYPE_NOT_STORED.setTokenized(true); OAK_TYPE_NOT_STORED.freeze(); + + Arrays.sort(TYPABLE_TAGS); } private final static class OakTextField extends Field { @@ -74,6 +90,10 @@ public final class FieldFactory { return new StringField(PATH, path, YES); } + public static Field newNameField(String name){ + return new StringField(NAME, name, NO); + } + public static Field newPropertyField(String name, String value, boolean tokenized, boolean stored) { if (tokenized) { @@ -86,4 +106,21 @@ public final class FieldFactory { return new TextField(FULLTEXT, value, NO); } + public static boolean canCreateTypedField(Type type){ + return Ints.contains(TYPABLE_TAGS, type.tag()); + } + + /** + * Date values are saved with 5 sec resolution + * @param date jcr data string + * @return date value in seconds + */ + public static long dateToLong(String date){ + //TODO Should we change the precision to 5 min resolution + //TODO make if configurable as part of property definition + long millis = ISO8601.parse(date).getTimeInMillis(); + long timeInSec = TimeUnit.MILLISECONDS.toSeconds(millis); + return timeInSec - timeInSec % 5; + } + } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java index c276274..b4cb0b0 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FieldNames.java @@ -38,6 +38,12 @@ public final class FieldNames { public static final String PATH = ":path"; /** + * Name of the field that contains the name of the node. + */ + public static final String NAME = ":name"; + + + /** * Name of the field that contains the fulltext index. */ public static final String FULLTEXT = ":fulltext"; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java new file mode 100644 index 0000000..eecec2b --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; + +import javax.annotation.CheckForNull; +import javax.jcr.PropertyType; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FULL_TEXT_ENABLED; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.FUNC_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_NAMES; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.STORE_NODE_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TEST_MODE_COST; + +public class IndexDefinition { + private static final Logger log = LoggerFactory.getLogger(IndexDefinition.class); + private final int propertyTypes; + + private final Set excludes; + + private final Set includes; + + private final boolean storageEnabled; + + private final boolean fullTextEnabled; + + private final boolean storeNodeName; + + private final double simulatedCostForTest; + + private final NodeBuilder definition; + + /** + * Index path and name would only be determined when IndexDefinition is created + * on the Reader side. On writer side this information is not present as we are + * only handed over the NodeBuilder + */ + + private final String defnPath; + + private final String name; + + private final String functionName; + + private final Map propDefns; + + public IndexDefinition(NodeState state, String path){ + this(new ReadOnlyBuilder(state), path); + } + + public IndexDefinition(NodeBuilder definition) { + this(definition, null); + } + + public IndexDefinition(NodeBuilder defn, String path) { + this.definition = defn; + this.defnPath = path; + this.name = path != null ? PathUtils.getName(path) : null; + PropertyState pst = defn.getProperty(INCLUDE_PROPERTY_TYPES); + if (pst != null) { + int types = 0; + for (String inc : pst.getValue(Type.STRINGS)) { + try { + types |= 1 << PropertyType.valueFromName(inc); + } catch (IllegalArgumentException e) { + log.warn("Unknown property type: " + inc); + } + } + this.propertyTypes = types; + } else { + this.propertyTypes = -1; + } + + this.excludes = getMultiProperty(defn, EXCLUDE_PROPERTY_NAMES); + this.includes = getMultiProperty(defn, INCLUDE_PROPERTY_NAMES); + this.fullTextEnabled = getOptionalValue(defn, FULL_TEXT_ENABLED, true); + + //Storage is disabled for non full text indexes + this.storageEnabled = this.fullTextEnabled && getOptionalValue(defn, EXPERIMENTAL_STORAGE, true); + this.storeNodeName = getOptionalValue(defn, STORE_NODE_NAME, false); + this.functionName = getOptionalValue(defn, FUNC_NAME, null); + this.simulatedCostForTest =defn.hasProperty(TEST_MODE_COST) ? + defn.getProperty(TEST_MODE_COST).getValue(Type.DOUBLE) : -1; + + + Map propDefns = Maps.newHashMap(); + for(String propName : includes){ + if(defn.hasChildNode(propName)){ + propDefns.put(propName, new PropertyDefinition(this, propName, defn.child(propName))); + } + } + this.propDefns = ImmutableMap.copyOf(propDefns); + } + + int getPropertyTypes() { + return propertyTypes; + } + + boolean includeProperty(String name) { + if(!includes.isEmpty()){ + return includes.contains(name); + } + return !excludes.contains(name); + } + + boolean includePropertyType(int type){ + //TODO Assumption that propertyTypes = -1 indicates no explicit property + //type defined hence all types would be included + if(propertyTypes < 0){ + return true; + } + return (propertyTypes & (1 << type)) != 0; + + } + + public boolean skipTokenization(String propertyName){ + if(!isFullTextEnabled()){ + return true; + } + return LuceneIndexHelper.skipTokenization(propertyName); + } + + /** + * Checks if a given property should be stored in the lucene index or not + */ + public boolean isStored(String name) { + return storageEnabled; + } + + public NodeBuilder getDefinition() { + return definition; + } + + public boolean isFullTextEnabled() { + return fullTextEnabled; + } + + public boolean isTestModeEnabled() { + return simulatedCostForTest > 0; + } + + public double getSimulatedCost() { + return simulatedCostForTest; + } + + public boolean isStoreNodeName() { + return storeNodeName; + } + + @CheckForNull + public String getDefinitionPath() { + return defnPath; + } + + public String getFunctionName() { + return functionName != null ? functionName : name; + } + + @CheckForNull + public String getName(){ + return name; + } + + @CheckForNull + public PropertyDefinition getPropDefn(String propName){ + return propDefns.get(propName); + } + + public boolean hasPropertyDefinition(String propName){ + return propDefns.containsKey(propName); + } + + //~------------------------------------------< Internal > + + private static boolean getOptionalValue(NodeBuilder definition, String propName, boolean defaultVal){ + PropertyState ps = definition.getProperty(propName); + return ps == null ? defaultVal : ps.getValue(Type.BOOLEAN); + } + + private static String getOptionalValue(NodeBuilder definition, String propName, String defaultVal){ + PropertyState ps = definition.getProperty(propName); + return ps == null ? defaultVal : ps.getValue(Type.STRING); + } + + private static Set getMultiProperty(NodeBuilder definition, String propName){ + PropertyState pse = definition.getProperty(propName); + return pse != null ? ImmutableSet.copyOf(pse.getValue(Type.STRINGS)) : Collections.emptySet(); + } +} diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java index 571e2b4..91fb5b8 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexNode.java @@ -68,7 +68,7 @@ class IndexNode { private final String name; - private final NodeState definition; + private final IndexDefinition definition; private final Directory directory; @@ -83,7 +83,7 @@ class IndexNode { IndexNode(String name, NodeState definition, Directory directory) throws IOException { this.name = name; - this.definition = definition; + this.definition = new IndexDefinition(new ReadOnlyBuilder(definition)); this.directory = directory; this.reader = DirectoryReader.open(directory); this.searcher = new IndexSearcher(reader); @@ -93,7 +93,7 @@ class IndexNode { return name; } - NodeState getDefinition() { + IndexDefinition getDefinition() { return definition; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java new file mode 100644 index 0000000..b2b82e8 --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.util.Collections; +import java.util.List; +import java.util.Set; + +import javax.annotation.CheckForNull; + +import org.apache.jackrabbit.oak.query.fulltext.FullTextExpression; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.query.QueryIndex; +import org.apache.lucene.index.IndexReader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.collect.Lists.newArrayListWithCapacity; +import static org.apache.jackrabbit.oak.spi.query.Filter.PropertyRestriction; +import static org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan; +import static org.apache.jackrabbit.oak.spi.query.QueryIndex.OrderEntry; + +public class IndexPlanner { + private static final Logger log = LoggerFactory.getLogger(IndexPlanner.class); + private final IndexDefinition defn; + private final String nativeFuncName; + private final IndexTracker tracker; + private final Filter filter; + private final List sortOrder; + private IndexNode indexNode; + + public IndexPlanner(IndexTracker tracker, IndexDefinition defn, + Filter filter, List sortOrder) { + this.tracker = tracker; + this.defn = defn; + this.nativeFuncName = LuceneIndex.getNativeFunctionName(defn); + this.filter = filter; + this.sortOrder = sortOrder; + } + + List getPlans() { + try { + IndexPlan.Builder plan = getPlan(); + + if(defn.isTestModeEnabled()){ + plan = getTestModePlan(plan); + } + + return plan != null ? Collections.singletonList(plan.build()) : + Collections.emptyList(); + } finally { + close(); + } + } + + private IndexPlan.Builder getTestModePlan(IndexPlan.Builder plan) { + //For now we support one full text index which indexes everything + //That index would be used irrespective of the constraints + if (plan == null && defn.isFullTextEnabled()){ + plan = defaultPlan(); + } + + //Lucene index yet not ready + if (plan == null){ + return null; + } + + return plan.setCostPerExecution(defn.getSimulatedCost()) + .setCostPerEntry(0); + } + + private void close(){ + if(indexNode != null){ + indexNode.release(); + } + } + + private IndexPlan.Builder getPlan() { + if (hasNativeFunction()) { + return defaultPlan(); + } + + FullTextExpression ft = filter.getFullTextConstraint(); + + if (ft != null) { + return getFullTextPlans(); + } + + List indexedProps = newArrayListWithCapacity(filter.getPropertyRestrictions().size()); + for(PropertyRestriction pr : filter.getPropertyRestrictions()){ + //Only those properties which are included and not tokenized + //can be managed by lucene for property restrictions + if(defn.includeProperty(pr.propertyName) + && defn.skipTokenization(pr.propertyName)){ + indexedProps.add(pr.propertyName); + } + } + + if(!indexedProps.isEmpty()){ + //TODO Need a way to have better cost estimate to indicate that + //this index can evaluate more propertyRestrictions natively (if more props are indexed) + //For now we reduce cost per entry + IndexPlan.Builder plan = defaultPlan(); + if(plan != null) { + return plan.setCostPerEntry(1.0 / indexedProps.size()); + } + } + + //TODO Support for path restrictions + //TODO support for NodeTypes + //TODO Support for property existence queries + //TODO support for nodeName queries + return null; + } + + private IndexPlan.Builder defaultPlan() { + IndexNode in = getIndexNode(); + if(in == null){ + return null; + } + return planBuilder().setEstimatedEntryCount(getReader().numDocs()); + } + + private IndexReader getReader() { + return getIndexNode().getSearcher().getIndexReader(); + } + + /** + * Checks if there is a native function for current index definition + */ + private boolean hasNativeFunction() { + return filter.getPropertyRestriction(nativeFuncName) != null; + } + + private IndexPlan.Builder getFullTextPlans() { + if (!defn.isFullTextEnabled()) { + return null; + } + Set relPaths = LuceneIndex.getRelativePaths(filter.getFullTextConstraint()); + if (relPaths.size() > 1) { + log.warn("More than one relative parent for query " + filter.getQueryStatement()); + // there are multiple "parents", as in + // "contains(a/x, 'hello') and contains(b/x, 'world')" + // return new MultiLuceneIndex(filter, root, relPaths).getCost(); + // MultiLuceneIndex not currently implemented + return null; + } + //TODO There might be multiple fulltext enabled indexes then we need to chose a default + //one + return defaultPlan(); + } + + private IndexNode getIndexNode(){ + if(indexNode == null){ + indexNode = tracker.acquireIndexNode(defn.getDefinitionPath()); + } + return indexNode; + } + + private IndexPlan.Builder planBuilder() { + return new IndexPlan.Builder() + .setCostPerExecution(1) // we're local. Low-cost + .setCostPerEntry(1) + .setFulltextIndex(defn.isFullTextEnabled()) + .setIncludesNodeData(false) // we should not include node data + .setFilter(filter) + .setSortOrder(createSortOrder()) + .setDelayed(true); //Lucene is always async + } + + @CheckForNull + private List createSortOrder() { + //TODO Refine later once we make mixed indexes having both + //full text and property index + if(defn.isFullTextEnabled()){ + return Collections.emptyList(); + } + + if(sortOrder == null){ + return null; + } + + List orderEntries = newArrayListWithCapacity(sortOrder.size()); + for(OrderEntry o : sortOrder){ + //sorting can only be done for known/configured properties + // and whose types are known + //TODO Can sorting be done for array properties + if(defn.includeProperty(o.getPropertyName()) + && o.getPropertyType() != null + && !o.getPropertyType().isArray()){ + orderEntries.add(o); //Lucene can manage any order desc/asc + } + } + return orderEntries; + } + +} diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java index 49749db..d9e4953 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkState; import static com.google.common.base.Predicates.in; import static com.google.common.base.Predicates.not; import static com.google.common.base.Predicates.notNull; -import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Lists.newArrayListWithCapacity; import static com.google.common.collect.Maps.filterKeys; import static com.google.common.collect.Maps.filterValues; @@ -32,17 +31,20 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstant import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; import java.io.IOException; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; +import javax.annotation.CheckForNull; + +import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.spi.commit.CompositeEditor; import org.apache.jackrabbit.oak.spi.commit.DefaultEditor; import org.apache.jackrabbit.oak.spi.commit.Editor; import org.apache.jackrabbit.oak.spi.commit.EditorDiff; import org.apache.jackrabbit.oak.spi.commit.SubtreeEditor; -import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -60,6 +62,8 @@ class IndexTracker { private volatile Map indices = emptyMap(); + private volatile Map definitions = emptyMap(); + synchronized void close() { Map indices = this.indices; this.indices = emptyMap(); @@ -75,17 +79,18 @@ class IndexTracker { synchronized void update(NodeState root) { Map original = indices; + Map originalDefns = definitions; final Map updates = newHashMap(); + final Map defnUpdates = newHashMap(); + + List editors = newArrayListWithCapacity(original.size() + 1); - List editors = newArrayListWithCapacity(original.size()); + editors.add(new SubtreeEditor(new IndexDefnEditor(defnUpdates, INDEX_DEFINITIONS_NAME) + , INDEX_DEFINITIONS_NAME)); for (Map.Entry entry : original.entrySet()) { final String path = entry.getKey(); final String name = entry.getValue().getName(); - List elements = newArrayList(); - Iterables.addAll(elements, PathUtils.elements(path)); - elements.add(INDEX_DEFINITIONS_NAME); - elements.add(name); editors.add(new SubtreeEditor(new DefaultEditor() { @Override public void leave(NodeState before, NodeState after) { @@ -97,7 +102,7 @@ class IndexTracker { log.error("Failed to open Lucene index at " + path, e); } } - }, elements.toArray(new String[elements.size()]))); + }, Iterables.toArray(PathUtils.elements(path), String.class))); } EditorDiff.process(CompositeEditor.compose(editors), this.root, root); @@ -118,6 +123,13 @@ class IndexTracker { } } } + + if(!defnUpdates.isEmpty()){ + definitions = ImmutableMap.builder() + .putAll(filterKeys(originalDefns, not(in(defnUpdates.keySet())))) + .putAll(filterValues(defnUpdates, notNull())) + .build(); + } } IndexNode acquireIndexNode(String path) { @@ -129,6 +141,22 @@ class IndexTracker { } } + Collection getDefinitions() { + return definitions.values(); + } + + IndexDefinition getDefinition(String path){ + return definitions.get(path); + } + + boolean hasDefinition(String path){ + return definitions.containsKey(path); + } + + int getDefinitionCount(){ + return definitions.size(); + } + Set getIndexNodePaths(){ return indices.keySet(); } @@ -145,24 +173,23 @@ class IndexTracker { NodeState node = root; for (String name : PathUtils.elements(path)) { - node = root.getChildNode(name); + node = node.getChildNode(name); } - node = node.getChildNode(INDEX_DEFINITIONS_NAME); + final String indexName = PathUtils.getName(path); try { - for (ChildNodeEntry child : node.getChildNodeEntries()) { - node = child.getNodeState(); - if (TYPE_LUCENE.equals(node.getString(TYPE_PROPERTY_NAME))) { - index = IndexNode.open(child.getName(), node); - if (index != null) { - checkState(index.acquire()); - indices = ImmutableMap.builder() - .putAll(indices) - .put(path, index) - .build(); - return index; - } + if (isLuceneIndexNode(node)) { + index = IndexNode.open(indexName, node); + if (index != null) { + checkState(index.acquire()); + indices = ImmutableMap.builder() + .putAll(indices) + .put(path, index) + .build(); + return index; } + } else if (node.exists()) { + log.warn("Cannot open Lucene Index at path {} as the index is not of type {}", path, TYPE_LUCENE); } } catch (IOException e) { log.error("Could not access the Lucene index at " + path, e); @@ -171,4 +198,54 @@ class IndexTracker { return null; } + private static boolean isLuceneIndexNode(NodeState node){ + return TYPE_LUCENE.equals(node.getString(TYPE_PROPERTY_NAME)); + } + + /** + * Editor to diff the child nodes of /oak:index so as to update the IndexDefinitions + */ + private static class IndexDefnEditor extends DefaultEditor { + final Map defnUpdates; + final String basePath; + + IndexDefnEditor(Map defnUpdates, String basePath) { + this.defnUpdates = defnUpdates; + this.basePath = basePath; + } + + @Override @CheckForNull + public Editor childNodeAdded(String name, NodeState after) throws CommitFailedException { + if(isLuceneIndexNode(after)) { + addEntry(name, after); + } + return null; + } + + @Override @CheckForNull + public Editor childNodeChanged(String name, NodeState before, NodeState after) throws CommitFailedException { + if(isLuceneIndexNode(after)) { + addEntry(name, after); + } + return null; + } + + @Override @CheckForNull + public Editor childNodeDeleted(String name, NodeState before) throws CommitFailedException { + if(isLuceneIndexNode(before)) { + defnUpdates.put(createPath(name), null); + } + return null; + } + + private void addEntry(String name, NodeState state){ + String path = createPath(name); + defnUpdates.put(path, new IndexDefinition(state, path)); + } + + private String createPath(String childName){ + return PathUtils.concat(basePath, childName); + } + } + } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index c2e7678..bbe34ba 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import static com.google.common.base.Preconditions.checkState; import static org.apache.jackrabbit.JcrConstants.JCR_MIXINTYPES; import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; +import static org.apache.jackrabbit.oak.api.Type.LONG; import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; import static org.apache.jackrabbit.oak.commons.PathUtils.getAncestorPath; @@ -26,12 +27,9 @@ import static org.apache.jackrabbit.oak.commons.PathUtils.getDepth; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newFulltextTerm; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; -import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization; import static org.apache.jackrabbit.oak.query.QueryImpl.JCR_PATH; import static org.apache.lucene.search.BooleanClause.Occur.MUST; import static org.apache.lucene.search.BooleanClause.Occur.MUST_NOT; @@ -49,6 +47,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import javax.annotation.CheckForNull; import javax.jcr.PropertyType; import com.google.common.collect.AbstractIterator; @@ -56,6 +55,7 @@ import com.google.common.collect.Queues; import com.google.common.collect.Sets; import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; import org.apache.jackrabbit.oak.query.QueryEngineSettings; @@ -92,10 +92,13 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -145,11 +148,11 @@ import org.slf4j.LoggerFactory; * @see QueryIndex * */ -public class LuceneIndex implements FulltextQueryIndex { +public class LuceneIndex implements FulltextQueryIndex, QueryIndex.AdvancedQueryIndex { private static final Logger LOG = LoggerFactory .getLogger(LuceneIndex.class); - public static final String NATIVE_QUERY_FUNCTION = "native*lucene"; + public static final String NATIVE_QUERY_FUNCTION1 = "native*lucene"; /** * Batch size for fetching results from Lucene queries. @@ -158,111 +161,39 @@ public class LuceneIndex implements FulltextQueryIndex { private final IndexTracker tracker; + private final IndexDefinition definition; + private final Analyzer analyzer; private final NodeAggregator aggregator; public LuceneIndex( - IndexTracker tracker, Analyzer analyzer, + IndexTracker tracker, IndexDefinition definition, Analyzer analyzer, NodeAggregator aggregator) { this.tracker = tracker; + this.definition = definition; this.analyzer = analyzer; this.aggregator = aggregator; } @Override public String getIndexName() { - return "lucene"; + return definition.getName(); } - @Override - public double getCost(Filter filter, NodeState root) { - FullTextExpression ft = filter.getFullTextConstraint(); - if (ft == null) { - // no full-text condition: don't use this index, - // as there might be a better one - return Double.POSITIVE_INFINITY; - } + //~----------------------------------------------< AdvanceQueryIndex > - IndexNode index = tracker.acquireIndexNode("/"); - if (index == null) { // unusable index - return Double.POSITIVE_INFINITY; - } - try { - Set relPaths = getRelativePaths(ft); - if (relPaths.size() > 1) { - LOG.warn("More than one relative parent for query " + filter.getQueryStatement()); - // there are multiple "parents", as in - // "contains(a/x, 'hello') and contains(b/x, 'world')" - return new MultiLuceneIndex(filter, root, relPaths).getCost(); - } - String parent = relPaths.iterator().next(); - if (parent.isEmpty()) { - // no relative properties - return 10; - } - // all relative properties have the same "parent", as in - // "contains(a/x, 'hello') and contains(a/y, 'world')" or - // "contains(a/x, 'hello') or contains(a/*, 'world')" - // TODO: proper cost calculation - // we assume this will cause more read operations, - // as we need to read the node and then the parent - return 15; - } finally { - index.release(); - } - } - - /** - * Get the set of relative paths of a full-text condition. For example, for - * the condition "contains(a/b, 'hello') and contains(c/d, 'world'), the set - * { "a", "c" } is returned. If there are no relative properties, then one - * entry is returned (the empty string). If there is no expression, then an - * empty set is returned. - * - * @param ft the full-text expression - * @return the set of relative paths (possibly empty) - */ - private static Set getRelativePaths(FullTextExpression ft) { - if (ft == null) { - // there might be no full-text constraint when using the - // LowCostLuceneIndexProvider which is used for testing - // TODO if the LowCostLuceneIndexProvider is removed, we should do - // the following instead: - - // throw new - // IllegalStateException("Lucene index is used even when no full-text conditions are used for filter " - // + filter); - - return Collections.emptySet(); - } - final HashSet relPaths = new HashSet(); - ft.accept(new FullTextVisitor.FullTextVisitorBase() { - - @Override - public boolean visit(FullTextTerm term) { - String p = term.getPropertyName(); - if (p == null) { - relPaths.add(""); - } else if (p.startsWith("../") || p.startsWith("./")) { - throw new IllegalArgumentException("Relative parent is not supported:" + p); - } else if (getDepth(p) > 1) { - String parent = getParentPath(p); - relPaths.add(parent); - } else { - relPaths.add(""); - } - return true; - } - }); - return relPaths; + @Override + public List getPlans(Filter filter, List sortOrder, NodeState rootState) { + return new IndexPlanner(tracker, definition, filter, sortOrder).getPlans(); } @Override - public String getPlan(Filter filter, NodeState root) { - IndexNode index = tracker.acquireIndexNode("/"); + public String getPlanDescription(IndexPlan plan, NodeState root) { + IndexNode index = acquireIndexNode(); checkState(index != null, "The Lucene index is not available"); try { + Filter filter = plan.getFilter(); FullTextExpression ft = filter.getFullTextConstraint(); Set relPaths = getRelativePaths(ft); if (relPaths.size() > 1) { @@ -272,18 +203,58 @@ public class LuceneIndex implements FulltextQueryIndex { // we only restrict non-full-text conditions if there is // no relative property in the full-text constraint boolean nonFullTextConstraints = parent.isEmpty(); - String plan = getQuery(filter, null, nonFullTextConstraints, analyzer, index.getDefinition()) + " ft:(" + ft + ")"; + StringBuilder sb = new StringBuilder("lucene:"); + sb.append(definition.getName()) + .append("(") + .append(definition.getDefinitionPath()) + .append(") "); + sb.append(getQuery(filter, null, nonFullTextConstraints, analyzer, definition)); + if(plan.getSortOrder() != null && !plan.getSortOrder().isEmpty()){ + sb.append(" ordering:").append(plan.getSortOrder()); + } + if (ft != null) { + sb.append(" ft:(").append(ft).append(")"); + } if (!parent.isEmpty()) { - plan += " parent:" + parent; + sb.append(" parent:").append(parent); } - return plan; + return sb.toString(); } finally { index.release(); } } @Override + public Cursor query(IndexPlan plan, NodeState root) { + final Filter filter = plan.getFilter(); + final Sort sort = getSort(plan.getSortOrder()); + return query(filter, sort, root); + } + + /** + * Though AdvanceQueryIndex is implemented need to implement + * older methods also for AggregateIndex to work + */ + @Override public Cursor query(final Filter filter, final NodeState root) { + return query(filter, null, root); + } + + @Override + public double getCost(Filter filter, NodeState root) { + throw new UnsupportedOperationException("Not supported as implementing AdvancedQueryIndex"); + } + + @Override + public String getPlan(Filter filter, NodeState root) { + List plans = getPlans(filter, Collections.emptyList(), root); + if(!plans.isEmpty()){ + return getPlanDescription(plans.get(0), root); + } + return "N/A"; + } + + private Cursor query(final Filter filter, final Sort sort, NodeState root) { FullTextExpression ft = filter.getFullTextConstraint(); Set relPaths = getRelativePaths(ft); if (relPaths.size() > 1) { @@ -311,6 +282,8 @@ public class LuceneIndex implements FulltextQueryIndex { private LuceneResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher) throws IOException { IndexReader reader = searcher.getIndexReader(); + //TODO Look into usage of field cache for retrieving the path + //instead of reading via reader if no of docs in index are limited PathStoredFieldVisitor visitor = new PathStoredFieldVisitor(); reader.document(doc.doc, visitor); String path = visitor.getPath(); @@ -346,17 +319,26 @@ public class LuceneIndex implements FulltextQueryIndex { private boolean loadDocs() { ScoreDoc lastDocToRecord = null; - IndexNode indexNode = tracker.acquireIndexNode("/"); + IndexNode indexNode = acquireIndexNode(); checkState(indexNode != null); try { IndexSearcher searcher = indexNode.getSearcher(); Query query = getQuery(filter, searcher.getIndexReader(), - nonFullTextConstraints, analyzer, indexNode.getDefinition()); + nonFullTextConstraints, analyzer, definition); TopDocs docs; + //TODO Support for sorting if (lastDoc != null) { - docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE); + if(sort == null) { + docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE); + } else { + docs = searcher.searchAfter(lastDoc, query, LUCENE_QUERY_BATCH_SIZE, sort); + } } else { - docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE); + if (sort == null) { + docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE); + } else { + docs = searcher.search(query, LUCENE_QUERY_BATCH_SIZE, sort); + } } for (ScoreDoc doc : docs.scoreDocs) { @@ -382,6 +364,86 @@ public class LuceneIndex implements FulltextQueryIndex { return new LucenePathCursor(itr, settings); } + private IndexNode acquireIndexNode() { + return tracker.acquireIndexNode(definition.getDefinitionPath()); + } + + + /** + * Get the set of relative paths of a full-text condition. For example, for + * the condition "contains(a/b, 'hello') and contains(c/d, 'world'), the set + * { "a", "c" } is returned. If there are no relative properties, then one + * entry is returned (the empty string). If there is no expression, then an + * empty set is returned. + * + * @param ft the full-text expression + * @return the set of relative paths (possibly empty) + */ + static Set getRelativePaths(FullTextExpression ft) { + if (ft == null) { + // there might be no full-text constraint when using the + // LowCostLuceneIndexProvider which is used for testing + // TODO if the LowCostLuceneIndexProvider is removed, we should do + // the following instead: + + // throw new + // IllegalStateException("Lucene index is used even when no full-text conditions are used for filter " + // + filter); + + return Collections.emptySet(); + } + final HashSet relPaths = new HashSet(); + ft.accept(new FullTextVisitor.FullTextVisitorBase() { + + @Override + public boolean visit(FullTextTerm term) { + String p = term.getPropertyName(); + if (p == null) { + relPaths.add(""); + } else if (p.startsWith("../") || p.startsWith("./")) { + throw new IllegalArgumentException("Relative parent is not supported:" + p); + } else if (getDepth(p) > 1) { + String parent = getParentPath(p); + relPaths.add(parent); + } else { + relPaths.add(""); + } + return true; + } + }); + return relPaths; + } + + private static Sort getSort(List sortOrder) { + if (sortOrder == null || sortOrder.isEmpty()) { + return null; + } + SortField[] fields = new SortField[sortOrder.size()]; + for (int i = 0; i < sortOrder.size(); i++) { + OrderEntry oe = sortOrder.get(i); + boolean reverse = oe.getOrder() == OrderEntry.Order.ASCENDING ? false : true; + fields[i] = new SortField(oe.getPropertyName(), toLuceneSortType(oe.getPropertyType()), reverse); + } + return new Sort(fields); + } + + private static SortField.Type toLuceneSortType(Type t) { + checkState(t != null, "Type cannot be null"); + checkState(!t.isArray(), "Array types are not supported"); + + switch (t.tag()) { + case PropertyType.LONG: + case PropertyType.DATE: + return SortField.Type.LONG; + case PropertyType.DOUBLE: + return SortField.Type.DOUBLE; + default: + //TODO Check about SortField.Type.STRING_VAL + return SortField.Type.STRING; + } + } + + /** * Get the Lucene query for the given filter. * @@ -395,7 +457,7 @@ public class LuceneIndex implements FulltextQueryIndex { * @return the Lucene query */ private static Query getQuery(Filter filter, IndexReader reader, - boolean nonFullTextConstraints, Analyzer analyzer, NodeState indexDefinition) { + boolean nonFullTextConstraints, Analyzer analyzer, IndexDefinition indexDefinition) { List qs = new ArrayList(); FullTextExpression ft = filter.getFullTextConstraint(); if (ft == null) { @@ -405,7 +467,7 @@ public class LuceneIndex implements FulltextQueryIndex { } else { qs.add(getFullTextQuery(ft, analyzer, reader)); } - PropertyRestriction pr = filter.getPropertyRestriction(NATIVE_QUERY_FUNCTION); + PropertyRestriction pr = filter.getPropertyRestriction(getNativeFunctionName(indexDefinition)); if (pr != null) { String query = String.valueOf(pr.first.getValue(pr.first.getType())); QueryParser queryParser = new QueryParser(VERSION, "", analyzer); @@ -442,8 +504,12 @@ public class LuceneIndex implements FulltextQueryIndex { return bq; } + static String getNativeFunctionName(IndexDefinition defn) { + return "native*" + defn.getFunctionName(); + } + private static void addNonFullTextConstraints(List qs, - Filter filter, IndexReader reader, Analyzer analyzer, NodeState indexDefinition) { + Filter filter, IndexReader reader, Analyzer analyzer, IndexDefinition indexDefinition) { if (!filter.matchesAllTypes()) { addNodeTypeConstraints(qs, filter); } @@ -487,6 +553,18 @@ public class LuceneIndex implements FulltextQueryIndex { if (pr.first == null && pr.last == null) { // ignore property existence checks, Lucene can't to 'property // is not null' queries (OAK-1208) + + //TODO May be this can be relaxed if we have an index which + //maintains the list of propertyName present in a node say + //against :propNames. Only configured set of properties would + //be + + //TODO Support for or clause i.e. pr.list + + //TODO Currently the propertyRestrictions are not in order i.e. FilterImpl + //maintains the restrictions in a hashmap. Having an order might help where + //first boolean clause should restrict the result set size for subsequent + //evaluation continue; } @@ -503,9 +581,12 @@ public class LuceneIndex implements FulltextQueryIndex { continue; } - if (skipTokenization(name)) { - qs.add(new TermQuery(new Term(name, pr.first - .getValue(STRING)))); + if (indexDefinition.skipTokenization(name) + && indexDefinition.includeProperty(pr.propertyName)) { + Query q = createQuery(pr, indexDefinition); + if(q != null) { + qs.add(q); + } continue; } @@ -524,28 +605,7 @@ public class LuceneIndex implements FulltextQueryIndex { } if (isLike) { - first = first.replace('%', WildcardQuery.WILDCARD_STRING); - first = first.replace('_', WildcardQuery.WILDCARD_CHAR); - - int indexOfWS = first.indexOf(WildcardQuery.WILDCARD_STRING); - int indexOfWC = first.indexOf(WildcardQuery.WILDCARD_CHAR); - int len = first.length(); - - if (indexOfWS == len || indexOfWC == len) { - // remove trailing "*" for prefixquery - first = first.substring(0, first.length() - 1); - if (JCR_PATH.equals(name)) { - qs.add(new PrefixQuery(newPathTerm(first))); - } else { - qs.add(new PrefixQuery(new Term(name, first))); - } - } else { - if (JCR_PATH.equals(name)) { - qs.add(new WildcardQuery(newPathTerm(first))); - } else { - qs.add(new WildcardQuery(new Term(name, first))); - } - } + qs.add(createLikeQuery(name, first)); continue; } @@ -572,6 +632,122 @@ public class LuceneIndex implements FulltextQueryIndex { } } + private static Query createLikeQuery(String name, String first) { + first = first.replace('%', WildcardQuery.WILDCARD_STRING); + first = first.replace('_', WildcardQuery.WILDCARD_CHAR); + + int indexOfWS = first.indexOf(WildcardQuery.WILDCARD_STRING); + int indexOfWC = first.indexOf(WildcardQuery.WILDCARD_CHAR); + int len = first.length(); + + if (indexOfWS == len || indexOfWC == len) { + // remove trailing "*" for prefixquery + first = first.substring(0, first.length() - 1); + if (JCR_PATH.equals(name)) { + return new PrefixQuery(newPathTerm(first)); + } else { + return new PrefixQuery(new Term(name, first)); + } + } else { + if (JCR_PATH.equals(name)) { + return new WildcardQuery(newPathTerm(first)); + } else { + return new WildcardQuery(new Term(name, first)); + } + } + } + + @CheckForNull + private static Query createQuery(PropertyRestriction pr, + IndexDefinition indexDefinition) { + //TODO Support for in query i.e. pr.list + int propType = pr.propertyType; + if (indexDefinition.hasPropertyDefinition(pr.propertyName)) { + propType = indexDefinition.getPropDefn(pr.propertyName).getPropertyType(); + } + switch (propType) { + case PropertyType.DATE: { + long first = pr.first != null ? FieldFactory.dateToLong(pr.first.getValue(Type.DATE)) : -1; + long last = pr.last != null ? FieldFactory.dateToLong(pr.last.getValue(Type.DATE)) : -1; + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding + && pr.lastIncluding) { + // [property]=[value] + return NumericRangeQuery.newLongRange(pr.propertyName, first, first, true, true); + } else if (pr.first != null && pr.last != null) { + return NumericRangeQuery.newLongRange(pr.propertyName, first, last, + pr.firstIncluding, pr.lastIncluding); + } else if (pr.first != null && pr.last == null) { + // '>' & '>=' use cases + return NumericRangeQuery.newLongRange(pr.propertyName, first, null, pr.firstIncluding, true); + } else if (pr.last != null && !pr.last.equals(pr.first)) { + // '<' & '<=' + return NumericRangeQuery.newLongRange(pr.propertyName, null, last, true, pr.lastIncluding); + } + } + case PropertyType.DOUBLE: { + Double first = pr.first != null ? pr.first.getValue(Type.DOUBLE) : null; + Double last = pr.last != null ? pr.last.getValue(Type.DOUBLE) : null; + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding + && pr.lastIncluding) { + // [property]=[value] + return NumericRangeQuery.newDoubleRange(pr.propertyName, first, first, true, true); + } else if (pr.first != null && pr.last != null) { + return NumericRangeQuery.newDoubleRange(pr.propertyName, first, last, + pr.firstIncluding, pr.lastIncluding); + } else if (pr.first != null && pr.last == null) { + // '>' & '>=' use cases + return NumericRangeQuery.newDoubleRange(pr.propertyName, first, null, pr.firstIncluding, true); + } else if (pr.last != null && !pr.last.equals(pr.first)) { + // '<' & '<=' + return NumericRangeQuery.newDoubleRange(pr.propertyName, null, last, true, pr.lastIncluding); + } + } + case PropertyType.LONG: { + Long first = pr.first != null ? pr.first.getValue(LONG) : null; + Long last = pr.last != null ? pr.last.getValue(LONG) : null; + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding + && pr.lastIncluding) { + // [property]=[value] + return NumericRangeQuery.newLongRange(pr.propertyName, first, first, true, true); + } else if (pr.first != null && pr.last != null) { + return NumericRangeQuery.newLongRange(pr.propertyName, first, last, + pr.firstIncluding, pr.lastIncluding); + } else if (pr.first != null && pr.last == null) { + // '>' & '>=' use cases + return NumericRangeQuery.newLongRange(pr.propertyName, first, null, pr.firstIncluding, true); + } else if (pr.last != null && !pr.last.equals(pr.first)) { + // '<' & '<=' + return NumericRangeQuery.newLongRange(pr.propertyName, null, last, true, pr.lastIncluding); + } + } + default: { + if (pr.isLike) { + return createLikeQuery(pr.propertyName, pr.first.getValue(STRING)); + } + + //TODO Confirm that all other types can be treated as string + String first = pr.first != null ? pr.first.getValue(STRING) : null; + String last = pr.last != null ? pr.last.getValue(STRING) : null; + if (pr.first != null && pr.first.equals(pr.last) && pr.firstIncluding + && pr.lastIncluding) { + // [property]=[value] + return new TermQuery(new Term(pr.propertyName, first)); + } else if (pr.first != null && pr.last != null) { + return TermRangeQuery.newStringRange(pr.propertyName, first, last, + pr.firstIncluding, pr.lastIncluding); + } else if (pr.first != null && pr.last == null) { + // '>' & '>=' use cases + return TermRangeQuery.newStringRange(pr.propertyName, first, null, pr.firstIncluding, true); + } else if (pr.last != null && !pr.last.equals(pr.first)) { + // '<' & '<=' + return TermRangeQuery.newStringRange(pr.propertyName, null, last, true, pr.lastIncluding); + } + } + } + //TODO Should not reach here + return null; + } + private static String tokenizeAndPoll(String token, Analyzer analyzer){ if (token != null) { List tokens = tokenize(token, analyzer); @@ -583,18 +759,16 @@ public class LuceneIndex implements FulltextQueryIndex { } private static boolean isExcludedProperty(PropertyRestriction pr, - NodeState definition) { + IndexDefinition definition) { String name = pr.propertyName; if (name.contains("/")) { // lucene cannot handle child-level property restrictions return true; } - // check name - for (String e : definition.getStrings(EXCLUDE_PROPERTY_NAMES)) { - if (e.equalsIgnoreCase(name)) { - return true; - } + // check + if(!definition.includeProperty(name)){ + return true; } // check type @@ -607,12 +781,7 @@ public class LuceneIndex implements FulltextQueryIndex { type = pr.list.get(0).getType().tag(); } if (type != null) { - boolean isIn = false; - for (String e : definition.getStrings(INCLUDE_PROPERTY_TYPES)) { - if (PropertyType.valueFromName(e) == type) { - isIn = true; - } - } + boolean isIn = definition.includePropertyType(type); if (!isIn) { return true; } @@ -639,6 +808,7 @@ public class LuceneIndex implements FulltextQueryIndex { private static void addNodeTypeConstraints(List qs, Filter filter) { BooleanQuery bq = new BooleanQuery(); + //TODO These condition should only be added if those propertyTypes are indexed for (String type : filter.getPrimaryTypes()) { bq.add(new TermQuery(new Term(JCR_PRIMARYTYPE, type)), SHOULD); } @@ -881,7 +1051,7 @@ public class LuceneIndex implements FulltextQueryIndex { public NodeAggregator getNodeAggregator() { return aggregator; } - + static class LuceneResultRow { final String path; final double score; diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index e41a606..98fd44b 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -39,6 +39,12 @@ public interface LuceneIndexConstants { */ String EXCLUDE_PROPERTY_NAMES = "excludePropertyNames"; + /** + * Only include properties with name in this set. If this property is defined + * then {@code excludePropertyNames} would be ignored + */ + String INCLUDE_PROPERTY_NAMES = "includePropertyNames"; + String PERSISTENCE_NAME = "persistence"; String PERSISTENCE_OAK = "repository"; @@ -54,4 +60,28 @@ public interface LuceneIndexConstants { */ String EXPERIMENTAL_STORAGE = "oak.experimental.storage"; + /** + * Determines if full text indexing is enabled for this index definition + */ + String FULL_TEXT_ENABLED = "fulltextEnabled"; + + String STORE_NODE_NAME = "storeNodeName"; + + /** + * Simulated cost used while test are being run. Used + */ + String TEST_MODE_COST = "simulatedCostForTest"; + + /** + * Name of the native function which this index supports. If not specified it + * defaults to index node name + */ + String FUNC_NAME = "functionName"; + + /** + * Type of the property being indexed defined as part of property definition + * under the given index definition + */ + String PROP_TYPE = "propertyType"; + } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 31e3eb0..58f1955 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -20,10 +20,10 @@ import static org.apache.jackrabbit.JcrConstants.JCR_DATA; import static org.apache.jackrabbit.oak.commons.PathUtils.concat; import static org.apache.jackrabbit.oak.commons.PathUtils.getName; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newFulltextField; +import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newNameField; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPathField; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPropertyField; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newPathTerm; -import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.skipTokenization; import java.io.IOException; import java.io.InputStream; @@ -42,7 +42,10 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.PrefixQuery; import org.apache.tika.metadata.Metadata; @@ -191,6 +194,8 @@ public class LuceneIndexEditor implements IndexEditor { } private Document makeDocument(String path, NodeState state, boolean isUpdate) throws CommitFailedException { + //TODO Possibly we can add support for compound properties like foo/bar + //i.e. support for relative path restrictions List fields = new ArrayList(); boolean dirty = false; for (PropertyState property : state.getProperties()) { @@ -202,13 +207,18 @@ public class LuceneIndexEditor implements IndexEditor { this.context.indexUpdate(); fields.addAll(newBinary(property, state)); dirty = true; + } else if(!context.isFullTextEnabled() + && FieldFactory.canCreateTypedField(property.getType())){ + dirty = addTypedFields(fields, property); } else { for (String value : property.getValue(Type.STRINGS)) { this.context.indexUpdate(); fields.add(newPropertyField(pname, value, - !skipTokenization(pname), + !context.skipTokenization(pname), context.isStored(pname))); - fields.add(newFulltextField(value)); + if (context.isFullTextEnabled()) { + fields.add(newFulltextField(value)); + } dirty = true; } } @@ -222,15 +232,46 @@ public class LuceneIndexEditor implements IndexEditor { Document document = new Document(); document.add(newPathField(path)); String name = getName(path); - if (name != null) { + if (context.isFullTextEnabled()) { document.add(newFulltextField(name)); } + if(context.getDefinition().isStoreNodeName()) { + document.add(newNameField(name)); + } for (Field f : fields) { document.add(f); } return document; } + private boolean addTypedFields(List fields, PropertyState property) throws CommitFailedException { + int tag = property.getType().tag(); + String name = property.getName(); + boolean fieldAdded = false; + for (int i = 0; i < property.count(); i++) { + Field f = null; + if (tag == Type.LONG.tag()) { + //TODO Distinguish fields which need to be used for search and for sort + //If a field is only used for Sort then it can be stored with less precision + f = new LongField(name, property.getValue(Type.LONG, i), Field.Store.NO); + } else if (tag == Type.DATE.tag()) { + String date = property.getValue(Type.DATE, i); + f = new LongField(name, FieldFactory.dateToLong(date), Field.Store.NO); + } else if (tag == Type.DOUBLE.tag()) { + f = new DoubleField(name, property.getValue(Type.DOUBLE, i), Field.Store.NO); + } else if (tag == Type.BOOLEAN.tag()) { + f = new StringField(name, property.getValue(Type.BOOLEAN, i).toString(), Field.Store.NO); + } + + if (f != null) { + this.context.indexUpdate(); + fields.add(f); + fieldAdded = true; + } + } + return fieldAdded; + } + private static boolean isVisible(String name) { return name.charAt(0) != ':'; } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 3107e21..5325b57 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -16,9 +16,6 @@ */ package org.apache.jackrabbit.oak.plugins.index.lucene; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXCLUDE_PROPERTY_NAMES; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.EXPERIMENTAL_STORAGE; -import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INCLUDE_PROPERTY_TYPES; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.INDEX_DATA_CHILD_NAME; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_PATH; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; @@ -27,12 +24,8 @@ import static org.apache.lucene.store.NoLockFactory.getNoLockFactory; import java.io.File; import java.io.IOException; import java.util.Calendar; -import java.util.Set; - -import javax.jcr.PropertyType; import org.apache.jackrabbit.oak.api.CommitFailedException; -import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -48,8 +41,6 @@ import org.apache.tika.parser.Parser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableSet; - public class LuceneIndexEditorContext { private static final Logger log = LoggerFactory @@ -63,6 +54,7 @@ public class LuceneIndexEditorContext { try { IndexWriterConfig config = new IndexWriterConfig(VERSION, analyzer); config.setMergeScheduler(new SerialMergeScheduler()); + //TODO Use default codec for index where full text index is not stored config.setCodec(new OakCodec()); return config; } finally { @@ -98,56 +90,30 @@ public class LuceneIndexEditorContext { private static final Parser parser = new AutoDetectParser(); - private final NodeBuilder definition; - private IndexWriter writer = null; - private final int propertyTypes; - - private final Set excludes; - private long indexedNodes; - private boolean storageEnabled = true; - private final IndexUpdateCallback updateCallback; + private final IndexDefinition definition; + + private final NodeBuilder definitionBuilder; + LuceneIndexEditorContext(NodeBuilder definition, Analyzer analyzer, IndexUpdateCallback updateCallback) { - this.definition = definition; + this.definitionBuilder = definition; + this.definition = new IndexDefinition(definition); this.config = getIndexWriterConfig(analyzer); - - PropertyState pst = definition.getProperty(INCLUDE_PROPERTY_TYPES); - if (pst != null) { - int types = 0; - for (String inc : pst.getValue(Type.STRINGS)) { - try { - types |= 1 << PropertyType.valueFromName(inc); - } catch (IllegalArgumentException e) { - log.warn("Unknown property type: " + inc); - } - } - this.propertyTypes = types; - } else { - this.propertyTypes = -1; - } - PropertyState pse = definition.getProperty(EXCLUDE_PROPERTY_NAMES); - if (pse != null) { - excludes = ImmutableSet.copyOf(pse.getValue(Type.STRINGS)); - } else { - excludes = ImmutableSet.of(); - } - PropertyState storage = definition.getProperty(EXPERIMENTAL_STORAGE); - storageEnabled = storage == null || storage.getValue(Type.BOOLEAN); this.indexedNodes = 0; this.updateCallback = updateCallback; } int getPropertyTypes() { - return propertyTypes; + return definition.getPropertyTypes(); } boolean includeProperty(String name) { - return !excludes.contains(name); + return definition.includeProperty(name); } Parser getParser() { @@ -156,7 +122,7 @@ public class LuceneIndexEditorContext { IndexWriter getWriter() throws IOException { if (writer == null) { - writer = new IndexWriter(newIndexDirectory(definition), config); + writer = new IndexWriter(newIndexDirectory(definition.getDefinition()), config); } return writer; } @@ -171,9 +137,9 @@ public class LuceneIndexEditorContext { //OAK-2029 Record the last updated status so //as to make IndexTracker detect changes when index //is stored in file system - NodeBuilder status = definition.child(":status"); + NodeBuilder status = definitionBuilder.child(":status"); status.setProperty("lastUpdated", ISO8601.format(Calendar.getInstance()), Type.DATE); - status.setProperty("indexedNodes",indexedNodes); + status.setProperty("indexedNodes", indexedNodes); } } @@ -195,7 +161,18 @@ public class LuceneIndexEditorContext { * */ public boolean isStored(String name) { - return storageEnabled; + return definition.isStored(name); + } + + public boolean isFullTextEnabled() { + return definition.isFullTextEnabled(); + } + + public boolean skipTokenization(String propertyName){ + return definition.skipTokenization(propertyName); } + public IndexDefinition getDefinition() { + return definition; + } } diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java index b13eba0..586dc5a 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java @@ -21,6 +21,7 @@ import java.util.List; import javax.annotation.Nonnull; +import com.google.common.collect.Lists; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.Observer; @@ -29,7 +30,6 @@ import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.lucene.analysis.Analyzer; -import com.google.common.collect.ImmutableList; /** * A provider for Lucene indexes. @@ -59,11 +59,15 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea @Override @Nonnull public List getQueryIndexes(NodeState nodeState) { - return ImmutableList. of(newLuceneIndex()); + List indexes = Lists.newArrayListWithCapacity(tracker.getDefinitionCount()); + for(IndexDefinition defn : tracker.getDefinitions()){ + indexes.add(newLuceneIndex(defn)); + } + return indexes; } - protected LuceneIndex newLuceneIndex() { - return new LuceneIndex(tracker, analyzer, aggregator); + protected LuceneIndex newLuceneIndex(IndexDefinition definition) { + return new LuceneIndex(tracker, definition, analyzer, aggregator); } /** diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java new file mode 100644 index 0000000..609c473 --- /dev/null +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import javax.jcr.PropertyType; + +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PropertyDefinition { + private static final Logger log = LoggerFactory.getLogger(PropertyDefinition.class); + private final String name; + private final NodeBuilder definition; + + private final int propertyType; + + public PropertyDefinition(IndexDefinition idxDefn, String name, NodeBuilder defn) { + this.name = name; + this.definition = defn; + + int type = PropertyType.UNDEFINED; + if(defn.hasProperty(LuceneIndexConstants.PROP_TYPE)){ + String typeName = defn.getString(LuceneIndexConstants.PROP_TYPE); + try{ + type = PropertyType.valueFromName(typeName); + } catch (IllegalArgumentException e){ + log.warn("Invalid property type {} for property {} in Index {}", typeName, name, idxDefn); + } + } + this.propertyType = type; + } + + public int getPropertyType() { + return propertyType; + } +} diff --git oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java index 95cc366..cdf1ee2 100644 --- oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java +++ oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/LuceneInitializerHelper.java @@ -22,6 +22,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHel import java.util.Set; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants; import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -37,6 +38,8 @@ public class LuceneInitializerHelper implements RepositoryInitializer { private String async = null; + private double simulatedCost = -1; + private Boolean storageEnabled; public LuceneInitializerHelper(String name) { @@ -75,19 +78,30 @@ public class LuceneInitializerHelper implements RepositoryInitializer { return this; } + public LuceneInitializerHelper simulatedCost(double simulatedCost) { + this.simulatedCost = simulatedCost; + return this; + } + @Override public void initialize(NodeBuilder builder) { + NodeBuilder idxBuilder = null; if (builder.hasChildNode(INDEX_DEFINITIONS_NAME) && builder.getChildNode(INDEX_DEFINITIONS_NAME).hasChildNode(name)) { // do nothing } else if (filePath == null) { - newLuceneIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), + idxBuilder = newLuceneIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), name, propertyTypes, excludes, async, storageEnabled); } else { - newLuceneFileIndexDefinition( + idxBuilder = newLuceneFileIndexDefinition( builder.child(INDEX_DEFINITIONS_NAME), name, propertyTypes, excludes, filePath, async); } + + if(idxBuilder != null + && simulatedCost > 0){ + idxBuilder.setProperty(LuceneIndexConstants.TEST_MODE_COST, simulatedCost); + } } } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java index 097c1f1..d5a58b1 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/jcr/LuceneOakRepositoryStub.java @@ -28,7 +28,6 @@ import javax.jcr.RepositoryException; import org.apache.jackrabbit.oak.plugins.index.aggregate.AggregateIndexProvider; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.aggregate.SimpleNodeAggregator; -import org.apache.jackrabbit.oak.plugins.index.lucene.LowCostLuceneIndexProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper; @@ -43,9 +42,9 @@ public class LuceneOakRepositoryStub extends OakTarMKRepositoryStub { @Override protected void preCreateRepository(Jcr jcr) { - LuceneIndexProvider provider = new LowCostLuceneIndexProvider(); + LuceneIndexProvider provider = new LuceneIndexProvider(); jcr.with( - new LuceneInitializerHelper("luceneGlobal", (Set) null)) + new LuceneInitializerHelper("luceneGlobal", (Set) null).simulatedCost(1e-3)) .with(AggregateIndexProvider.wrap(provider.with(getNodeAggregator()))) .with((Observer) provider) .with(new LuceneIndexEditorProvider()); diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java new file mode 100644 index 0000000..bf7baca --- /dev/null +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import com.google.common.collect.ImmutableSet; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static javax.jcr.PropertyType.TYPENAME_STRING; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.STORE_NODE_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.newLuceneIndexDefinition; +import static org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class IndexTrackerTest { + private NodeState root = INITIAL_CONTENT; + private NodeBuilder builder = root.builder(); + private IndexTracker tracker = new IndexTracker(); + + @Test + public void detectDefinitionChanges() throws Exception{ + NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); + newLuceneIndexDefinition(index, "lucene", + ImmutableSet.of(TYPENAME_STRING)); + + tracker.update(builder.getNodeState()); + assertEquals(1, tracker.getDefinitions().size()); + assertTrue(tracker.hasDefinition("oak:index/lucene")); + + IndexDefinition dfn = tracker.getDefinition("oak:index/lucene"); + assertTrue(dfn.includePropertyType(Type.STRING.tag())); + assertFalse(dfn.isStoreNodeName()); + + newLuceneIndexDefinition(index, "lucene2", + ImmutableSet.of(TYPENAME_STRING)); + tracker.update(builder.getNodeState()); + assertEquals(2, tracker.getDefinitions().size()); + assertTrue(tracker.hasDefinition("oak:index/lucene2")); + + builder.child(INDEX_DEFINITIONS_NAME).child("lucene").setProperty(STORE_NODE_NAME, true); + tracker.update(builder.getNodeState()); + dfn = tracker.getDefinition("oak:index/lucene"); + assertTrue(dfn.isStoreNodeName()); + } +} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java deleted file mode 100644 index 3cc3e84..0000000 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LowCostLuceneIndexProvider.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.jackrabbit.oak.plugins.index.lucene; - -import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; -import org.apache.jackrabbit.oak.spi.query.Filter; -import org.apache.jackrabbit.oak.spi.state.NodeState; -import org.apache.lucene.analysis.Analyzer; - -/** - * A LuceneIndexProvider that return a LuceneIndex with a really low cost, so - * that it tries to guarantee its usage in the queries - * - */ -public class LowCostLuceneIndexProvider extends LuceneIndexProvider { - - @Override - protected LuceneIndex newLuceneIndex() { - return new LowCostLuceneIndex(tracker, analyzer, aggregator); - } - - private static class LowCostLuceneIndex extends LuceneIndex { - - public LowCostLuceneIndex(IndexTracker tracker, Analyzer analyzer, NodeAggregator aggregator) { - super(tracker, analyzer, aggregator); - } - - @Override - public double getCost(Filter filter, NodeState root) { - return 1e-3; - } - } -} diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java index 1f13614..112882c 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexAggregationTest.java @@ -56,7 +56,7 @@ public class LuceneIndexAggregationTest extends AbstractQueryTest { @Override protected ContentRepository createRepository() { - LowCostLuceneIndexProvider provider = new LowCostLuceneIndexProvider(); + LuceneIndexProvider provider = new LuceneIndexProvider(); return new Oak() .with(new InitialContent()) .with(new OpenSecurityProvider()) diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexExclusionQueryTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexExclusionQueryTest.java index e762ccc..39374e6 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexExclusionQueryTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexExclusionQueryTest.java @@ -58,7 +58,7 @@ public class LuceneIndexExclusionQueryTest extends AbstractQueryTest { @Override protected ContentRepository createRepository() { - LowCostLuceneIndexProvider provider = new LowCostLuceneIndexProvider(); + LuceneIndexProvider provider = new LuceneIndexProvider(); return new Oak().with(new InitialContent()) .with(new OpenSecurityProvider()) .with((QueryIndexProvider) provider) diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java index a25e59d..a77f1d5 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexQueryTest.java @@ -45,13 +45,15 @@ public class LuceneIndexQueryTest extends AbstractQueryTest { @Override protected void createTestIndexNode() throws Exception { Tree index = root.getTree("/"); - createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE); + Tree idxTree = createTestIndexNode(index, LuceneIndexConstants.TYPE_LUCENE); + idxTree.setProperty(LuceneIndexConstants.FUNC_NAME, "lucene"); + idxTree.setProperty(LuceneIndexConstants.TEST_MODE_COST, 1e-3); root.commit(); } @Override protected ContentRepository createRepository() { - LowCostLuceneIndexProvider provider = new LowCostLuceneIndexProvider(); + LuceneIndexProvider provider = new LuceneIndexProvider(); return new Oak().with(new InitialContent()) .with(new OpenSecurityProvider()) .with((QueryIndexProvider) provider) @@ -273,7 +275,7 @@ public class LuceneIndexQueryTest extends AbstractQueryTest { @Test public void testRepSimilarAsNativeQuery() throws Exception { - String nativeQueryString = "select [jcr:path] from [nt:base] where " + + String nativeQueryString = "select [jcr:path] from [nt:base] where " + "native('lucene', 'mlt?stream.body=/test/a&mlt.fl=:path&mlt.mindf=0&mlt.mintf=0')"; Tree test = root.getTree("/").addChild("test"); test.addChild("a").setProperty("text", "Hello World"); diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java index f0d6211..8876450 100644 --- oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java @@ -80,9 +80,11 @@ public class LuceneIndexTest { @Test public void testLucene() throws Exception { NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); - newLuceneIndexDefinition(index, "lucene", + NodeBuilder defnBuilder = newLuceneIndexDefinition(index, "lucene", ImmutableSet.of(TYPENAME_STRING)); + IndexDefinition defn = new IndexDefinition(defnBuilder, "/oak:index/lucene"); + NodeState before = builder.getNodeState(); builder.setProperty("foo", "bar"); NodeState after = builder.getNodeState(); @@ -91,7 +93,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - QueryIndex queryIndex = new LuceneIndex(tracker, analyzer, null); + QueryIndex queryIndex = new LuceneIndex(tracker, defn, analyzer, null); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -105,9 +107,11 @@ public class LuceneIndexTest { @Test public void testLuceneLazyCursor() throws Exception { NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); - newLuceneIndexDefinition(index, "lucene", + NodeBuilder defnBuilder = newLuceneIndexDefinition(index, "lucene", ImmutableSet.of(TYPENAME_STRING)); + IndexDefinition defn = new IndexDefinition(defnBuilder, "/oak:index/lucene"); + NodeState before = builder.getNodeState(); builder.setProperty("foo", "bar"); @@ -121,7 +125,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - QueryIndex queryIndex = new LuceneIndex(tracker, analyzer, null); + QueryIndex queryIndex = new LuceneIndex(tracker, defn, analyzer, null); FilterImpl filter = createFilter(NT_BASE); filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); @@ -139,9 +143,11 @@ public class LuceneIndexTest { @Test public void testLucene2() throws Exception { NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); - newLuceneIndexDefinition(index, "lucene", + NodeBuilder defnBuilder = newLuceneIndexDefinition(index, "lucene", ImmutableSet.of(TYPENAME_STRING)); + IndexDefinition defn = new IndexDefinition(defnBuilder, "/oak:index/lucene"); + NodeState before = builder.getNodeState(); builder.setProperty("foo", "bar"); builder.child("a").setProperty("foo", "bar"); @@ -154,7 +160,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - QueryIndex queryIndex = new LuceneIndex(tracker, analyzer, null); + QueryIndex queryIndex = new LuceneIndex(tracker, defn, analyzer, null); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -172,9 +178,11 @@ public class LuceneIndexTest { @Test public void testLucene3() throws Exception { NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); - newLuceneIndexDefinition(index, "lucene", + NodeBuilder defnBuilder = newLuceneIndexDefinition(index, "lucene", ImmutableSet.of(TYPENAME_STRING)); + IndexDefinition defn = new IndexDefinition(defnBuilder, "/oak:index/lucene"); + NodeState before = builder.getNodeState(); builder.setProperty("foo", "bar"); builder.child("a").setProperty("foo", "bar"); @@ -188,7 +196,7 @@ public class LuceneIndexTest { IndexTracker tracker = new IndexTracker(); tracker.update(indexed); - QueryIndex queryIndex = new LuceneIndex(tracker, analyzer, null); + QueryIndex queryIndex = new LuceneIndex(tracker, defn, analyzer, null); FilterImpl filter = createFilter(NT_BASE); // filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty("foo", Operator.EQUAL, @@ -254,6 +262,7 @@ public class LuceneIndexTest { idxb.setProperty(PERSISTENCE_NAME, PERSISTENCE_FILE); idxb.setProperty(PERSISTENCE_PATH, getIndexDir()); + IndexDefinition defn = new IndexDefinition(idxb, "/oak:index/lucene"); nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); builder = nodeStore.getRoot().builder(); @@ -261,17 +270,17 @@ public class LuceneIndexTest { NodeState indexed = nodeStore.merge(builder, HOOK, CommitInfo.EMPTY); - assertQuery(tracker, indexed, "foo", "bar"); + assertQuery(tracker, defn, indexed, "foo", "bar"); builder = nodeStore.getRoot().builder(); builder.setProperty("foo2", "bar2"); indexed = nodeStore.merge(builder, HOOK, CommitInfo.EMPTY); - assertQuery(tracker, indexed, "foo2", "bar2"); + assertQuery(tracker, defn, indexed, "foo2", "bar2"); } - private void assertQuery(IndexTracker tracker, NodeState indexed, String key, String value){ - QueryIndex queryIndex = new LuceneIndex(tracker, analyzer, null); + private void assertQuery(IndexTracker tracker, IndexDefinition defn, NodeState indexed, String key, String value){ + QueryIndex queryIndex = new LuceneIndex(tracker, defn, analyzer, null); FilterImpl filter = createFilter(NT_BASE); filter.restrictPath("/", Filter.PathRestriction.EXACT); filter.restrictProperty(key, Operator.EQUAL, @@ -287,4 +296,7 @@ public class LuceneIndexTest { return dir.getAbsolutePath(); } + + //TODO Test for Sorting + //TODO Test for path restrictions } diff --git oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiLuceneIndexQueryTest.java oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiLuceneIndexQueryTest.java new file mode 100644 index 0000000..3863086 --- /dev/null +++ oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiLuceneIndexQueryTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.util.List; +import java.util.Set; + +import javax.jcr.PropertyType; + +import com.google.common.collect.ImmutableSet; +import org.apache.jackrabbit.JcrConstants; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.Result; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneInitializerHelper; +import org.apache.jackrabbit.oak.plugins.memory.PropertyStates; +import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent; +import org.apache.jackrabbit.oak.query.AbstractQueryTest; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.junit.Test; +import org.junit.matchers.JUnitMatchers; + +import static com.google.common.collect.ImmutableSet.of; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.junit.Assert.assertThat; +import static org.junit.matchers.JUnitMatchers.containsString; +import static org.junit.matchers.JUnitMatchers.hasItems; + +public class MultiLuceneIndexQueryTest extends AbstractQueryTest { + + @Override + protected void createTestIndexNode() throws Exception { + setTraversalEnabled(false); + } + + @Override + protected ContentRepository createRepository() { + LuceneIndexProvider provider = new LuceneIndexProvider(); + return new Oak() + .with(new InitialContent()) + .with(new LuceneInitializerHelper("luceneGlobal", (Set) null)) + .with(new OpenSecurityProvider()) + .with((QueryIndexProvider) provider) + .with((Observer) provider) + .with(new LuceneIndexEditorProvider()) + .createContentRepository(); + } + + @Test + public void indexSelection() throws Exception{ + createIndex("test1", of("propa", "propb")); + createIndex("test2", of("propc")); + + Tree test = root.getTree("/").addChild("test"); + test.addChild("a").setProperty("propa", "foo"); + test.addChild("b").setProperty("propa", "foo"); + test.addChild("c").setProperty("propa", "foo2"); + test.addChild("d").setProperty("propc", "foo"); + test.addChild("e").setProperty("propd", "foo"); + root.commit(); + + String propaQuery = "select [jcr:path] from [nt:base] where [propa] = 'foo'"; + assertThat(explain(propaQuery), containsString("lucene:test1")); + assertThat(explain("select [jcr:path] from [nt:base] where [propc] = 'foo'"), containsString("lucene:test2")); + + assertThat(exq(propaQuery), hasItems("/test/a", "/test/b")); + assertThat(exq("select [jcr:path] from [nt:base] where [propa] = 'foo2'"), hasItems("/test/c")); + assertThat(exq("select [jcr:path] from [nt:base] where [propc] = 'foo'"), hasItems("/test/d")); + } + + @Test + public void rangeQueries() throws Exception{ + Tree idx = createIndex("test1", of("propa", "propb")); + Tree propIdx = idx.addChild("propa"); + propIdx.setProperty(LuceneIndexConstants.PROP_TYPE, PropertyType.TYPENAME_LONG); + root.commit(); + + Tree test = root.getTree("/").addChild("test"); + test.addChild("a").setProperty("propa", 10); + test.addChild("b").setProperty("propa", 20); + test.addChild("c").setProperty("propa", 30); + test.addChild("c").setProperty("propb", "foo"); + test.addChild("d").setProperty("propb", "foo"); + root.commit(); + + assertThat(exq("select [jcr:path] from [nt:base] where [propa] >= 20"), hasItems("/test/b", "/test/c")); + assertThat(exq("select [jcr:path] from [nt:base] where [propa] = 20"), hasItems("/test/b")); + assertThat(exq("select [jcr:path] from [nt:base] where [propa] <= 20"), hasItems("/test/b", "/test/a")); + assertThat(exq("select [jcr:path] from [nt:base] where [propa] < 20"), hasItems("/test/a")); + + } + + private String explain(String query){ + String explain = "explain " + query; + return executeQuery(explain, "JCR-SQL2").get(0); + } + + private List exq(String stmt){ + return executeQuery(stmt, "JCR-SQL2"); + } + + private Tree createIndex(String name, Set propNames) throws CommitFailedException { + Tree index = root.getTree("/"); + Tree def = index.addChild(INDEX_DEFINITIONS_NAME).addChild(name); + def.setProperty(JcrConstants.JCR_PRIMARYTYPE, + INDEX_DEFINITIONS_NODE_TYPE, Type.NAME); + def.setProperty(TYPE_PROPERTY_NAME, LuceneIndexConstants.TYPE_LUCENE); + def.setProperty(REINDEX_PROPERTY_NAME, true); + def.setProperty(LuceneIndexConstants.FULL_TEXT_ENABLED, false); + def.setProperty(PropertyStates.createProperty(LuceneIndexConstants.INCLUDE_PROPERTY_NAMES, propNames, Type.STRINGS)); + root.commit(); + return root.getTree("/").getChild(INDEX_DEFINITIONS_NAME).getChild(name); + } + +}