diff --git a/oak-lucene/pom.xml b/oak-lucene/pom.xml index bccf0ac..6e3c7e0 100644 --- a/oak-lucene/pom.xml +++ b/oak-lucene/pom.xml @@ -102,9 +102,13 @@ - org.apache.jackrabbit.oak.plugins.index.lucene, - org.apache.jackrabbit.oak.plugins.index.lucene.util + org.apache.jackrabbit.oak.plugins.index.lucene, + org.apache.jackrabbit.oak.plugins.index.lucene.util, + org.apache.jackrabbit.oak.plugins.index.lucene.score, + <_exportcontents> + org.apache.lucene.*;version=${lucene.version} + org.apache.lucene.sandbox.*;resolution:=optional, * diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index e955fa3..21170b3 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -168,6 +168,8 @@ class IndexDefinition implements Aggregate.AggregateMapper{ private final Analyzer analyzer; + private final String scorerProviderName; + private final Map analyzers; public IndexDefinition(NodeState root, NodeState defn) { @@ -225,6 +227,7 @@ class IndexDefinition implements Aggregate.AggregateMapper{ this.indexesAllTypes = areAllTypesIndexed(); this.analyzers = collectAnalyzers(defn); this.analyzer = createAnalyzer(); + this.scorerProviderName = getOptionalValue(defn, LuceneIndexConstants.PROP_SCORER_PROVIDER, null); } public boolean isFullTextEnabled() { @@ -305,6 +308,10 @@ class IndexDefinition implements Aggregate.AggregateMapper{ return analyzer; } + public String getScorerProviderName(){ + return scorerProviderName; + } + @Override public String toString() { return "IndexDefinition : " + indexName; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 61a3a14..e89019a 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -130,6 +130,8 @@ public interface LuceneIndexConstants { String PROP_ORDERED = "ordered"; + String PROP_SCORER_PROVIDER = "scorerProviderName"; + /** * Integer property indicating that LuceneIndex should be * used in compat mode to specific version diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java index 216cfdb..a80030f 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProvider.java @@ -23,6 +23,7 @@ import javax.annotation.Nonnull; import org.apache.jackrabbit.oak.plugins.index.aggregate.AggregateIndex; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.spi.commit.CommitInfo; import org.apache.jackrabbit.oak.spi.commit.Observer; import org.apache.jackrabbit.oak.spi.query.QueryIndex; @@ -42,12 +43,19 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea protected volatile NodeAggregator aggregator = null; + ScorerProviderFactory scorerFactory; + public LuceneIndexProvider() { this(new IndexTracker()); } public LuceneIndexProvider(IndexTracker tracker) { + this(tracker, ScorerProviderFactory.DEFAULT); + } + + public LuceneIndexProvider(IndexTracker tracker, ScorerProviderFactory scorerFactory) { this.tracker = tracker; + this.scorerFactory = scorerFactory; } public void close() { @@ -73,7 +81,7 @@ public class LuceneIndexProvider implements QueryIndexProvider, Observer, Closea } protected LucenePropertyIndex newLucenePropertyIndex() { - return new LucenePropertyIndex(tracker); + return new LucenePropertyIndex(tracker, scorerFactory); } /** diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index 4f0f587..b7e37f0 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -39,6 +39,7 @@ import org.apache.felix.scr.annotations.ReferencePolicyOption; import org.apache.jackrabbit.oak.commons.PropertiesUtil; import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.spi.commit.Observer; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.whiteboard.Registration; @@ -99,13 +100,16 @@ public class LuceneIndexProviderService { private WhiteboardExecutor executor; + @Reference + ScorerProviderFactory scorerFactory; + @Activate private void activate(BundleContext bundleContext, Map config) throws NotCompliantMBeanException { initializeFactoryClassLoaders(getClass().getClassLoader()); whiteboard = new OsgiWhiteboard(bundleContext); - indexProvider = new LuceneIndexProvider(createTracker(bundleContext, config)); + indexProvider = new LuceneIndexProvider(createTracker(bundleContext, config), scorerFactory); initializeLogging(config); initialize(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index 098b4d1..f8ed766 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -41,6 +41,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.aggregate.NodeAggregator; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; import org.apache.jackrabbit.oak.query.QueryEngineSettings; import org.apache.jackrabbit.oak.query.QueryImpl; @@ -70,6 +71,7 @@ import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.queries.CustomScoreQuery; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; @@ -101,7 +103,6 @@ import static org.apache.jackrabbit.JcrConstants.JCR_PRIMARYTYPE; import static org.apache.jackrabbit.oak.api.Type.LONG; import static org.apache.jackrabbit.oak.api.Type.STRING; import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; -import static org.apache.jackrabbit.oak.commons.PathUtils.getName; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.NATIVE_SORT_ORDER; @@ -170,8 +171,16 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati protected final IndexTracker tracker; + private final ScorerProviderFactory scorerProviderFactory; + public LucenePropertyIndex(IndexTracker tracker) { this.tracker = tracker; + this.scorerProviderFactory = ScorerProviderFactory.DEFAULT; + } + + public LucenePropertyIndex(IndexTracker tracker, ScorerProviderFactory factory) { + this.tracker = tracker; + this.scorerProviderFactory = factory; } @Override @@ -301,6 +310,13 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati try { IndexSearcher searcher = indexNode.getSearcher(); Query query = getQuery(plan, searcher.getIndexReader()); + + CustomScoreQuery customScoreQuery = getCustomScoreQuery(plan, query); + + if (customScoreQuery != null) { + query = customScoreQuery; + } + TopDocs docs; long time = System.currentTimeMillis(); if (lastDoc != null) { @@ -516,6 +532,17 @@ public class LucenePropertyIndex implements AdvancedQueryIndex, QueryIndex, Nati return bq; } + private CustomScoreQuery getCustomScoreQuery(IndexPlan plan, Query subQuery) { + PlanResult planResult = pr(plan); + IndexDefinition idxDef = planResult.indexDefinition; + String providerName = idxDef.getScorerProviderName(); + if (scorerProviderFactory != null && providerName != null) { + return scorerProviderFactory.getScorerProvider(providerName) + .createCustomScoreQuery(subQuery); + } + return null; + } + private static void addNonFullTextConstraints(List qs, IndexPlan plan, IndexReader reader) { Filter filter = plan.getFilter(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProvider.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProvider.java new file mode 100644 index 0000000..9d0e56c --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProvider.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.score; + +import org.apache.lucene.queries.CustomScoreQuery; +import org.apache.lucene.search.Query; + +public interface ScorerProvider { + String getName(); + CustomScoreQuery createCustomScoreQuery(Query subQuery); +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProviderFactory.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProviderFactory.java new file mode 100644 index 0000000..e24af69 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/ScorerProviderFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.score; + +public interface ScorerProviderFactory { + ScorerProviderFactory DEFAULT = new ScorerProviderFactory() { + @Override + public ScorerProvider getScorerProvider(String name) { + return null; + } + }; + + ScorerProvider getScorerProvider(String name); +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/impl/ScorerProviderFactoryImpl.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/impl/ScorerProviderFactoryImpl.java new file mode 100644 index 0000000..bf167ab --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/score/impl/ScorerProviderFactoryImpl.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.lucene.score.impl; + +import org.apache.felix.scr.annotations.Activate; +import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Deactivate; +import org.apache.felix.scr.annotations.Reference; +import org.apache.felix.scr.annotations.ReferenceCardinality; +import org.apache.felix.scr.annotations.ReferencePolicy; +import org.apache.felix.scr.annotations.Service; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component(metatype = false, immediate = true) +@Service(value = ScorerProviderFactory.class) +@Reference(name = "ScorerProvider", + policy = ReferencePolicy.DYNAMIC, + cardinality = ReferenceCardinality.OPTIONAL_MULTIPLE, + referenceInterface = ScorerProvider.class, + bind = "bindScorerProvider", + unbind = "unbindScorerProvider" +) +public class ScorerProviderFactoryImpl implements ScorerProviderFactory { + + private Map scorerProviderMap = + new ConcurrentHashMap(); + + @Activate + private void activate() { + scorerProviderMap.clear(); + } + + @Deactivate + private void deactivate() { + scorerProviderMap.clear(); + } + + @Override + public ScorerProvider getScorerProvider(String scorerName) { + return scorerProviderMap.get(scorerName); + } + + private void bindScorerProvider(ScorerProvider provider) { + scorerProviderMap.put(provider.getName(), provider); + } + + private void unbindScorerProvider(ScorerProvider provider) { + scorerProviderMap.remove(provider.getName()); + } +} diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java index cee4ca0..409f444 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java @@ -17,8 +17,10 @@ package org.apache.jackrabbit.oak.plugins.index.lucene; import java.io.File; +import java.io.IOException; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Set; import javax.annotation.Nonnull; @@ -31,6 +33,7 @@ import static com.google.common.collect.Iterators.transform; import static com.google.common.collect.Lists.newArrayList; import static com.google.common.collect.Sets.newHashSet; import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor; +import static java.util.Arrays.asList; import static javax.jcr.PropertyType.TYPENAME_STRING; import static junit.framework.Assert.assertEquals; import static junit.framework.Assert.assertFalse; @@ -48,6 +51,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstant import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_FILE; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_NAME; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PERSISTENCE_PATH; +import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.useV2; import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.newLuceneIndexDefinition; import static org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexHelper.newLucenePropertyIndexDefinition; import static org.apache.jackrabbit.oak.plugins.nodetype.NodeTypeConstants.JCR_NODE_TYPES; @@ -56,10 +60,13 @@ import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvancedQueryIndex; import static org.apache.jackrabbit.oak.spi.query.QueryIndex.IndexPlan; import com.google.common.base.Function; +import com.google.common.collect.Maps; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore; import org.apache.jackrabbit.oak.query.QueryEngineSettings; import org.apache.jackrabbit.oak.query.ast.Operator; @@ -79,6 +86,12 @@ import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.CustomScoreProvider; +import org.apache.lucene.queries.CustomScoreQuery; +import org.apache.lucene.search.Query; import org.junit.After; import org.junit.Test; @@ -317,6 +330,71 @@ public class LuceneIndexTest { } @Test + public void customScoreQuery() throws Exception{ + NodeBuilder nb = newLuceneIndexDefinition(builder.child(INDEX_DEFINITIONS_NAME), "lucene", + of(TYPENAME_STRING)); + TestUtil.useV2(nb); + nb.setProperty(LuceneIndexConstants.PROP_SCORER_PROVIDER, "testScorer"); + + NodeState before = builder.getNodeState(); + builder.child("a").setProperty("jcr:createdBy", "bar bar"); + builder.child("b").setProperty("jcr:createdBy", "foo bar"); + NodeState after = builder.getNodeState(); + NodeState indexed = HOOK.processCommit(before, after,CommitInfo.EMPTY); + IndexTracker tracker = new IndexTracker(); + tracker.update(indexed); + + SimpleScorerFactory factory = new SimpleScorerFactory(); + ScorerProvider provider = new ScorerProvider() { + + String scorerName = "testScorer"; + @Override + public String getName() { + return scorerName; + } + @Override + public CustomScoreQuery createCustomScoreQuery(Query query) { + return new ModifiedCustomScoreQuery(query); + } + + class ModifiedCustomScoreQuery extends CustomScoreQuery { + private Query query; + public ModifiedCustomScoreQuery(Query query) { + super(query); + this.query = query; + } + + @Override + public CustomScoreProvider getCustomScoreProvider(AtomicReaderContext context) { + return new CustomScoreProvider(context) { + public float customScore(int doc, float subQueryScore, float valSrcScore) { + AtomicReader atomicReader = context.reader(); + try { + Document document = atomicReader.document(doc); + // boosting docs created by foo + String fieldValue = document.get("full:jcr:createdBy"); + if (fieldValue != null && fieldValue.contains("foo")) { + valSrcScore *= 2.0; + } + } catch (IOException e) { + return subQueryScore * valSrcScore; + } + return subQueryScore * valSrcScore; + } + }; + } + } + }; + + factory.providers.put(provider.getName(), provider); + AdvancedQueryIndex queryIndex = new LucenePropertyIndex(tracker, factory); + + FilterImpl filter = createFilter(NT_BASE); + filter.setFullTextConstraint(new FullTextTerm(null, "bar", false, false, null)); + assertFilter(filter, queryIndex, indexed, asList("/b", "/a"), true); + } + + @Test public void testTokens() { Analyzer analyzer = LuceneIndexConstants.ANALYZER; assertEquals(ImmutableList.of("parent", "child"), @@ -498,9 +576,37 @@ public class LuceneIndexTest { return paths; } + private static List assertFilter(Filter filter, AdvancedQueryIndex queryIndex, + NodeState indexed, List expected, boolean ordered) { + if (!ordered) { + return assertFilter(filter, queryIndex, indexed, expected); + } + + List plans = queryIndex.getPlans(filter, null, indexed); + Cursor cursor = queryIndex.query(plans.get(0), indexed); + + List paths = newArrayList(); + while (cursor.hasNext()) { + paths.add(cursor.next().getPath()); + } + for (String p : expected) { + assertTrue("Expected path " + p + " not found", paths.contains(p)); + } + assertEquals("Result set size is different", expected.size(), paths.size()); + return paths; + } + private String getIndexDir(){ File dir = new File("target", "indexdir"+System.nanoTime()); dirs.add(dir); return dir.getAbsolutePath(); } + + private static class SimpleScorerFactory implements ScorerProviderFactory { + final Map providers = Maps.newHashMap(); + @Override + public ScorerProvider getScorerProvider(String name) { + return providers.get(name); + } + } }