diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java index 454ea47034..ea7023d3ac 100644 --- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java +++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java @@ -21,6 +21,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -46,15 +47,22 @@ import com.google.common.base.Joiner; import com.google.common.base.Predicate; import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; import com.google.common.util.concurrent.MoreExecutors; import org.apache.commons.io.FileUtils; import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.api.jmx.IndexStatsMBean; import org.apache.jackrabbit.oak.fixture.JcrCreator; import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture; import org.apache.jackrabbit.oak.fixture.RepositoryFixture; import org.apache.jackrabbit.oak.jcr.Jcr; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl; import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.IndexPathService; +import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl; import org.apache.jackrabbit.oak.plugins.index.IndexUtils; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; @@ -64,15 +72,19 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; -import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.PropertyRule; +import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver; import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer; import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Registration; import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; import org.apache.jackrabbit.oak.stats.Clock; @@ -82,7 +94,11 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE; import static org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants.NT_OAK_UNSTRUCTURED; +import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay; public class HybridIndexTest extends AbstractTest { enum Status { @@ -113,10 +129,12 @@ public Status next(){ private int numOfIndexes = Integer.getInteger("numOfIndexes", 10); private int refreshDeltaMillis = Integer.getInteger("refreshDeltaMillis", 1000); private int asyncInterval = Integer.getInteger("asyncInterval", 5); + private int cleanerIntervalInSecs = Integer.getInteger("cleanerIntervalInSecs", 10); private int queueSize = Integer.getInteger("queueSize", 1000); private boolean hybridIndexEnabled = Boolean.getBoolean("hybridIndexEnabled"); private boolean dumpStats = Boolean.getBoolean("dumpStats"); private boolean useOakCodec = Boolean.parseBoolean(System.getProperty("useOakCodec", "true")); + private boolean syncIndexing = Boolean.parseBoolean(System.getProperty("syncIndexing", "false")); private String indexingMode = System.getProperty("indexingMode", "nrt"); private boolean searcherEnabled = Boolean.parseBoolean(System.getProperty("searcherEnabled", "true")); @@ -139,6 +157,8 @@ public Status next(){ private final Logger log = LoggerFactory.getLogger(getClass()); private final ExecutorService executorService = MoreExecutors.getExitingExecutorService( (ThreadPoolExecutor) Executors.newFixedThreadPool(5)); + private final List regs = new ArrayList<>(); + private BackgroundObserver backgroundObserver; public HybridIndexTest(File workDir, StatisticsProvider statsProvider) { @@ -153,17 +173,21 @@ public HybridIndexTest(File workDir, StatisticsProvider statsProvider) { @Override public Jcr customize(Oak oak) { Jcr jcr = new Jcr(oak); - prepareLuceneIndexer(workDir); + whiteboard = oak.getWhiteboard(); + prepareLuceneIndexer(workDir, getNodeStore(oak)); + + backgroundObserver = new BackgroundObserver(luceneIndexProvider, executorService, 5); + jcr.with((QueryIndexProvider) luceneIndexProvider) - .with((Observer) luceneIndexProvider) - .with(luceneEditorProvider); + .with(backgroundObserver) + .with(luceneEditorProvider) + .with(new NodeTypeIndexFixerInitializer()); if (hybridIndexEnabled) { jcr.with(localIndexObserver); indexInitializer = new LuceneIndexInitializer(); } - whiteboard = oak.getWhiteboard(); jcr.with(indexInitializer); //Configure the default global fulltext index as it impacts @@ -238,6 +262,19 @@ protected void afterSuite() throws Exception { //and before NodeStore teardown getAsyncIndexUpdate().close(); + if (backgroundObserver != null){ + backgroundObserver.close(); + } + + int sleepCount = 0; + while (backgroundObserver.getMBean().getQueueSize()> 0 && ++sleepCount < 100) { + TimeUnit.MILLISECONDS.sleep(100); + } + + for (Registration r : regs) { + r.unregister(); + } + //Close hybrid stuff after async is closed if (hybridIndexEnabled){ queue.close(); @@ -248,9 +285,10 @@ protected void afterSuite() throws Exception { FileUtils.deleteDirectory(indexCopierDir); } System.out.printf("numOfIndexes: %d, refreshDeltaMillis: %d, asyncInterval: %d, queueSize: %d , " + - "hybridIndexEnabled: %s, indexingMode: %s, useOakCodec: %s %n", + "hybridIndexEnabled: %s, indexingMode: %s, useOakCodec: %s, cleanerIntervalInSecs: %d, " + + "syncIndexing: %s %n", numOfIndexes, refreshDeltaMillis, asyncInterval, queueSize, hybridIndexEnabled, - indexingMode, useOakCodec); + indexingMode, useOakCodec, cleanerIntervalInSecs, syncIndexing); if (dumpStats) { dumpStats(); @@ -281,6 +319,9 @@ protected String comment() { if (useOakCodec){ commentElements.add("oakCodec"); } + if (syncIndexing) { + commentElements.add("sync"); + } } else { commentElements.add("property"); } @@ -319,7 +360,7 @@ private String randomStatus() { return status.name(); } - private void prepareLuceneIndexer(File workDir) { + private void prepareLuceneIndexer(File workDir, NodeStore nodeStore) { try { indexCopierDir = createTemporaryFolderIn(workDir); copier = new IndexCopier(executorService, indexCopierDir, true); @@ -327,11 +368,16 @@ private void prepareLuceneIndexer(File workDir) { throw new RuntimeException(e); } + IndexPathService indexPathService = new IndexPathServiceImpl(nodeStore); + AsyncIndexInfoService asyncIndexInfoService = new AsyncIndexInfoServiceImpl(nodeStore); + nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP); MountInfoProvider mip = Mounts.defaultMountInfoProvider(); LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier); + IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory); + luceneIndexProvider = new LuceneIndexProvider(tracker); luceneEditorProvider = new LuceneIndexEditorProvider(copier, tracker, @@ -343,6 +389,13 @@ private void prepareLuceneIndexer(File workDir) { localIndexObserver = new LocalIndexObserver(queue, statsProvider); luceneEditorProvider.setIndexingQueue(queue); + if (syncIndexing) { + PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService); + regs.add(scheduleWithFixedDelay(whiteboard, cleaner, + cleanerIntervalInSecs, true, true)); + } + + Thread.setDefaultUncaughtExceptionHandler((t, e) -> log.warn("Uncaught exception", e)); } @@ -375,6 +428,16 @@ private static File createTemporaryFolderIn(File parentFolder) throws IOExceptio return createdFolder; } + private static NodeStore getNodeStore(Oak oak) { + try { + Field f = Oak.class.getDeclaredField("store"); + f.setAccessible(true); + return (NodeStore) f.get(oak); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + private class PropertyIndexInitializer implements RepositoryInitializer { @Override @@ -388,8 +451,11 @@ public void initialize(@Nonnull NodeBuilder builder) { private void addPropIndexDefn(NodeBuilder parent, String propName){ try { - IndexUtils.createIndexDefinition(parent, propName, false, + NodeBuilder idx = IndexUtils.createIndexDefinition(parent, propName, false, singleton(propName), null, "property", null); + if ( propName.equals(indexedPropName)) { + idx.setProperty("tags", singletonList("fooIndex"), Type.STRINGS); + } } catch (RepositoryException e) { throw new RuntimeException(e); } @@ -405,7 +471,10 @@ public void initialize(@Nonnull NodeBuilder builder) { IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder(); defnBuilder.evaluatePathRestrictions(); defnBuilder.async("async", indexingMode, "async"); - defnBuilder.indexRule("nt:base").property(indexedPropName).propertyIndex(); + PropertyRule pr = defnBuilder.indexRule("nt:base").property(indexedPropName).propertyIndex(); + if (syncIndexing) { + pr.sync(); + } if (useOakCodec) { defnBuilder.codec("oakCodec"); } @@ -415,6 +484,7 @@ public void initialize(@Nonnull NodeBuilder builder) { } oakIndex.setChildNode(indexedPropName, defnBuilder.build()); + oakIndex.child(indexedPropName).setProperty("tags", singletonList("fooIndex"), Type.STRINGS); } } @@ -433,6 +503,35 @@ public void initialize(@Nonnull NodeBuilder builder) { } } + private class NodeTypeIndexFixerInitializer implements RepositoryInitializer { + + @Override + public void initialize(@Nonnull NodeBuilder builder) { + //Due to OAK-1150 currently all nodes get indexed + //With explicit list on those nodes would be indexed + NodeBuilder nodetype = builder.getChildNode("oak:index").getChildNode("nodetype"); + if (nodetype.exists()) { + List nodetypes = Lists.newArrayList(); + if (nodetype.hasProperty(DECLARING_NODE_TYPES)){ + nodetypes = Lists.newArrayList(nodetype.getProperty(DECLARING_NODE_TYPES).getValue(Type.STRINGS)); + } + + if (nodetypes.isEmpty()) { + nodetypes.add(INDEX_DEFINITIONS_NODE_TYPE); + nodetypes.add("rep:Authorizable"); + nodetype.setProperty(DECLARING_NODE_TYPES, nodetypes, Type.NAMES); + nodetype.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true); + } + } + + //Disable counter index to disable traversal + NodeBuilder counter = builder.getChildNode("oak:index").getChildNode("counter"); + if (counter.exists()) { + counter.setProperty("type", "disabled"); + } + } + } + private class Searcher implements Runnable { final Session session = loginWriter(); int resultSize = 0; @@ -448,7 +547,8 @@ public void run() { private void run0() throws RepositoryException { session.refresh(false); QueryManager qm = session.getWorkspace().getQueryManager(); - Query q = qm.createQuery("select * from [nt:base] where [" + indexedPropName + "] = $status", Query.JCR_SQL2); + Query q = qm.createQuery("select * from [nt:base] where [" + indexedPropName + "] = $status " + + "option(index tag fooIndex)", Query.JCR_SQL2); q.bindValue("status", session.getValueFactory().createValue(randomStatus())); QueryResult result = q.execute(); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoService.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoService.java index 48d1a9af86..c93070fd11 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoService.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoService.java @@ -19,6 +19,8 @@ package org.apache.jackrabbit.oak.plugins.index; +import java.util.Map; + import javax.annotation.CheckForNull; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -52,4 +54,33 @@ */ @CheckForNull AsyncIndexInfo getInfo(String name, NodeState root); + + /** + * Returns the lastIndexUpto time in millis for each indexing lane + * for current root state + * + * @return map with lane name as key and lastIndexUpto in millis as value + */ + Map getIndexedUptoPerLane(); + + /** + * Returns the lastIndexUpto time in millis for each indexing lane + * for given root state + * + * @return map with lane name as key and lastIndexUpto in millis as value + */ + Map getIndexedUptoPerLane(NodeState root); + + /** + * Determines if any index lane has completed any indexing cycle between given + * two repository states + * + * @param before before state of root node + * @param after after state of root node + * @return true if any indexing lane has completed any indexing cycle i.e. its + * lastIndexTo time has changed + */ + default boolean hasIndexerUpdatedForAnyLane(NodeState before, NodeState after) { + return !getIndexedUptoPerLane(before).equals(getIndexedUptoPerLane(after)); + } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImpl.java index 89b899bd3a..bc84d8cd8e 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImpl.java @@ -25,6 +25,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import com.google.common.collect.ImmutableMap; import org.apache.felix.scr.annotations.Component; import org.apache.felix.scr.annotations.Reference; import org.apache.felix.scr.annotations.ReferenceCardinality; @@ -87,7 +88,7 @@ public AsyncIndexInfo getInfo(String name) { public AsyncIndexInfo getInfo(String name, NodeState root) { NodeState async = getAsyncState(root); if (async.hasProperty(name)) { - long lastIndexedTo = getDateAsMillis(async.getProperty(AsyncIndexUpdate.lastIndexedTo(name))); + long lastIndexedTo = getLastIndexedTo(name, async); long leaseEnd = -1; boolean running = false; if (async.hasProperty(AsyncIndexUpdate.leasify(name))) { @@ -100,6 +101,25 @@ public AsyncIndexInfo getInfo(String name, NodeState root) { return null; } + @Override + public Map getIndexedUptoPerLane() { + return getIndexedUptoPerLane(nodeStore.getRoot()); + } + + @Override + public Map getIndexedUptoPerLane(NodeState root) { + ImmutableMap.Builder builder = new ImmutableMap.Builder(); + NodeState async = getAsyncState(root); + for (PropertyState ps : async.getProperties()) { + String name = ps.getName(); + if (AsyncIndexUpdate.isAsyncLaneName(name)) { + long lastIndexedTo = getLastIndexedTo(name, async); + builder.put(name, lastIndexedTo); + } + } + return builder.build(); + } + private NodeState getAsyncState(NodeState root) { return root.getChildNode(AsyncIndexUpdate.ASYNC); } @@ -112,6 +132,10 @@ protected void unbindStatsMBeans(IndexStatsMBean mBean) { statsMBeans.remove(mBean.getName()); } + private static long getLastIndexedTo(String name, NodeState async) { + return getDateAsMillis(async.getProperty(AsyncIndexUpdate.lastIndexedTo(name))); + } + private static long getDateAsMillis(PropertyState ps) { if (ps == null) { return -1; diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java index 5dc0405130..fa0fbc9499 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/AsyncIndexInfoServiceImplTest.java @@ -19,11 +19,16 @@ package org.apache.jackrabbit.oak.plugins.index; +import java.util.Map; import java.util.Set; import com.google.common.collect.ImmutableSet; import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; import org.junit.Test; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -76,4 +81,37 @@ public void info() throws Exception{ assertNull(info2.getStatsMBean()); } + @Test + public void indexedUpto() throws Exception{ + AsyncIndexUpdate async = new AsyncIndexUpdate("async", store, provider); + async.run(); + + AsyncIndexUpdate async2 = new AsyncIndexUpdate("foo-async", store, provider); + async2.run(); + + Map result = service.getIndexedUptoPerLane(); + + assertFalse(result.isEmpty()); + assertTrue(result.get("async") > -1); + assertTrue(result.get("foo-async") > -1); + } + + @Test + public void asyncStateChanged() throws Exception{ + AsyncIndexUpdate async = new AsyncIndexUpdate("async", store, provider); + async.run(); + + AsyncIndexUpdate async2 = new AsyncIndexUpdate("foo-async", store, provider); + async2.run(); + + NodeState root = store.getRoot(); + assertFalse(service.hasIndexerUpdatedForAnyLane(root, root)); + + NodeBuilder builder = store.getRoot().builder(); + builder.child(":async").setProperty(AsyncIndexUpdate.lastIndexedTo("async"), 42L); + store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + + assertTrue(service.hasIndexerUpdatedForAnyLane(root, store.getRoot())); + } + } \ No newline at end of file diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/Aggregate.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/Aggregate.java index 0dd01c7632..3eca2536b2 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/Aggregate.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/Aggregate.java @@ -411,6 +411,10 @@ public boolean aggregatesProperty(String name) { public String toString() { return propertyDefinition.toString(); } + + public PropertyDefinition getPropertyDefinition() { + return propertyDefinition; + } } public static interface ResultCollector { @@ -464,6 +468,8 @@ public PropertyIncludeResult(PropertyState propertyState, PropertyDefinition pd, public interface AggregateRoot { void markDirty(); + + String getPath(); } public static class Matcher { @@ -609,6 +615,19 @@ public void markRootDirty() { rootState.root.markDirty(); } + public String getRootPath() { + return rootState.root.getPath(); + } + + public String getMatchedPath(){ + checkArgument(status == Status.MATCH_FOUND); + return currentPath; + } + + public Include getCurrentInclude(){ + return currentInclude; + } + public Status getStatus() { return status; } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java index 2eb6733f7b..fdea434563 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java @@ -266,6 +266,8 @@ @Nullable private final String[] indexTags; + private final boolean syncPropertyIndexes; + //~--------------------------------------------------------< Builder > public static Builder newBuilder(NodeState root, NodeState defn, String indexPath){ @@ -402,6 +404,7 @@ private IndexDefinition(NodeState root, NodeState defn, IndexFormatVersion versi this.spellcheckEnabled = evaluateSpellcheckEnabled(); this.nrtIndexMode = supportsNRTIndexing(defn); this.syncIndexMode = supportsSyncIndexing(defn); + this.syncPropertyIndexes = definedRules.stream().anyMatch(ir -> !ir.syncProps.isEmpty()); } public NodeState getDefinitionNodeState() { @@ -555,6 +558,10 @@ public boolean isSyncIndexingEnabled() { return syncIndexMode; } + public boolean hasSyncPropertyDefinitions() { + return syncPropertyIndexes; + } + /** * Check if the index definition is fresh of some index has happened * @@ -871,6 +878,7 @@ public int getNumberOfTopFacets() { private final List functionRestrictions; private final List notNullCheckEnabledProperties; private final List nodeScopeAnalyzedProps; + private final List syncProps; private final boolean indexesAllNodesOfMatchingType; private final boolean nodeNameIndexed; @@ -897,9 +905,10 @@ public int getNumberOfTopFacets() { List functionRestrictions = newArrayList(); List existentProperties = newArrayList(); List nodeScopeAnalyzedProps = newArrayList(); + List syncProps = newArrayList(); List propIncludes = newArrayList(); this.propConfigs = collectPropConfigs(config, namePatterns, propIncludes, nonExistentProperties, - existentProperties, nodeScopeAnalyzedProps, functionRestrictions); + existentProperties, nodeScopeAnalyzedProps, functionRestrictions, syncProps); this.propAggregate = new Aggregate(nodeTypeName, propIncludes); this.aggregate = combine(propAggregate, nodeTypeName); @@ -913,6 +922,7 @@ public int getNumberOfTopFacets() { this.propertyIndexEnabled = hasAnyPropertyIndexConfigured(); this.indexesAllNodesOfMatchingType = areAlMatchingNodeByTypeIndexed(); this.nodeNameIndexed = evaluateNodeNameIndexed(config); + this.syncProps = ImmutableList.copyOf(syncProps); validateRuleDefinition(); } @@ -942,6 +952,7 @@ public int getNumberOfTopFacets() { this.nodeFullTextIndexed = aggregate.hasNodeAggregates() || original.nodeFullTextIndexed; this.indexesAllNodesOfMatchingType = areAlMatchingNodeByTypeIndexed(); this.nodeNameIndexed = original.nodeNameIndexed; + this.syncProps = original.syncProps; } /** @@ -1109,7 +1120,8 @@ public boolean isBasedOnNtBase(){ List nonExistentProperties, List existentProperties, List nodeScopeAnalyzedProps, - List functionRestrictions) { + List functionRestrictions, + List syncProps) { Map propDefns = newHashMap(); NodeState propNode = config.getChildNode(LuceneIndexConstants.PROP_NODE); @@ -1165,6 +1177,10 @@ public boolean isBasedOnNtBase(){ && !pd.isRegexp){ nodeScopeAnalyzedProps.add(pd); } + + if (pd.sync) { + syncProps.add(pd); + } } } return ImmutableMap.copyOf(propDefns); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java index a233fe603a..9b8f5ef84e 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlanner.java @@ -291,6 +291,14 @@ static void setUseActualEntryCount(boolean useActualEntryCount) { result.enableNodeNameRestriction(); } + if (sortOrder.isEmpty()) { + boolean uniqueIndexFound = planForSyncIndexes(); + if (uniqueIndexFound) { + //For unique index there would be at max 1 entry + plan.setEstimatedEntryCount(1); + } + } + return plan.setCostPerEntry(definition.getCostPerEntry() / costPerEntryFactor); } @@ -618,6 +626,44 @@ private static PropertyDefinition getSimpleProperty(IndexingRule indexingRule, S return indexingRule.getConfig(name); } + private boolean planForSyncIndexes() { + //If no sync index involved then return right away + if (!definition.hasSyncPropertyDefinitions() || result.propDefns.isEmpty()) { + return false; + } + + List unique = newArrayList(); + List nonUnique = newArrayList(); + + for (PropertyRestriction pr : filter.getPropertyRestrictions()) { + String propertyName = result.getPropertyName(pr); + PropertyDefinition pd = result.propDefns.get(pr.propertyName); + + if (pd != null) { + PropertyIndexResult e = new PropertyIndexResult(pd, propertyName, pr); + if (pd.unique) { + unique.add(e); + } else { + nonUnique.add(e); + } + } + } + + //TODO NodeType restrictions + //Pick the first index (if multiple). For unique its fine + //For non unique we can probably later add support for cost + //based selection + boolean uniqueIndexFound = false; + if (!unique.isEmpty()) { + result.propertyIndexResult = unique.get(0); + uniqueIndexFound = true; + } else if (!nonUnique.isEmpty()) { + result.propertyIndexResult = nonUnique.get(0); + } + + return uniqueIndexFound; + } + private boolean canEvalPathRestrictions(IndexingRule rule) { //Opt out if one is looking for all children for '/' as its equivalent to //NO_RESTRICTION @@ -804,7 +850,12 @@ private static boolean nodeScopedTerm(String propertyName) { final IndexDefinition indexDefinition; final IndexingRule indexingRule; private final List sortedProperties = newArrayList(); + + //Map of actual property name as present in our property definitions private final Map propDefns = newHashMap(); + + //Map of property restriction name -> property definition name + //like 'jcr:content/status' -> 'status' private final Map relPropMapping = newHashMap(); private boolean nonFullTextConstraints; @@ -814,6 +865,7 @@ private static boolean nodeScopedTerm(String propertyName) { private boolean nodeTypeRestrictions; private boolean nodeNameRestriction; private boolean uniquePathsRequired = true; + private PropertyIndexResult propertyIndexResult; public PlanResult(String indexPath, IndexDefinition defn, IndexingRule indexingRule) { this.indexPath = indexPath; @@ -881,6 +933,15 @@ public boolean evaluateNodeTypeRestriction() { public boolean evaluateNodeNameRestriction() {return nodeNameRestriction;} + @CheckForNull + public PropertyIndexResult getPropertyIndexResult() { + return propertyIndexResult; + } + + public boolean hasPropertyIndexResult(){ + return propertyIndexResult != null; + } + private void setParentPath(String relativePath){ parentPathSegment = "/" + relativePath; if (relativePath.isEmpty()){ @@ -909,4 +970,16 @@ private void disableUniquePaths(){ uniquePathsRequired = false; } } + + public static class PropertyIndexResult { + final PropertyDefinition pd; + final String propertyName; + final PropertyRestriction pr; + + public PropertyIndexResult(PropertyDefinition pd, String propertyName, PropertyRestriction pr) { + this.pd = pd; + this.propertyName = propertyName; + this.pr = pr; + } + } } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java index d729f62e48..7ad825b567 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java @@ -39,6 +39,7 @@ import com.google.common.collect.Sets; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; @@ -69,6 +70,8 @@ private NodeState root = EMPTY_NODE; + private AsyncIndexInfoService asyncIndexInfoService; + private volatile Map indices = emptyMap(); private volatile boolean refresh; @@ -114,7 +117,21 @@ public synchronized void update(final NodeState root) { } } + public void setAsyncIndexInfoService(AsyncIndexInfoService asyncIndexInfoService) { + this.asyncIndexInfoService = asyncIndexInfoService; + } + + AsyncIndexInfoService getAsyncIndexInfoService() { + return asyncIndexInfoService; + } + private synchronized void diffAndUpdate(final NodeState root) { + if (asyncIndexInfoService != null && !asyncIndexInfoService.hasIndexerUpdatedForAnyLane(this.root, root)) { + log.trace("No changed detected in async indexer state. Skipping further diff"); + this.root = root; + return; + } + Map original = indices; final Map updates = newHashMap(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java index c1a572b61b..88da912642 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java @@ -37,6 +37,7 @@ import org.apache.jackrabbit.oak.api.Result.SizePrecision; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SpellcheckHelper; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; @@ -1203,32 +1204,4 @@ public long getSize(SizePrecision precision, long max) { return estimatedSize = sizeEstimator.getSize(); } } - - private static class PathStoredFieldVisitor extends StoredFieldVisitor { - - private String path; - private boolean pathVisited; - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (PATH.equals(fieldInfo.name)) { - return Status.YES; - } - return pathVisited ? Status.STOP : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) - throws IOException { - if (PATH.equals(fieldInfo.name)) { - path = value; - pathVisited = true; - } - } - - public String getPath() { - return path; - } - } - } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java index 724565388d..0ea867c3fe 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexConstants.java @@ -156,6 +156,16 @@ public static IndexingMode from(String indexingMode){ String PROP_WEIGHT = "weight"; /** + * Boolean property in property definition to mark sync properties + */ + String PROP_SYNC = "sync"; + + /** + * Boolean property in property definition to mark unique properties + */ + String PROP_UNIQUE = "unique"; + + /** * Integer property indicating that LuceneIndex should be * used in compat mode to specific version */ diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java index 2e474486f5..41abbb89ab 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor.java @@ -150,6 +150,11 @@ public void leave(NodeState before, NodeState after) } if (parent == null) { + PropertyUpdateCallback callback = context.getPropertyUpdateCallback(); + if (callback != null) { + callback.done(); + } + try { context.closeWriter(); } catch (IOException e) { @@ -168,6 +173,7 @@ public void leave(NodeState before, NodeState after) public void propertyAdded(PropertyState after) { markPropertyChanged(after.getName()); checkAggregates(after.getName()); + propertyUpdated(null, after); } @Override @@ -175,6 +181,7 @@ public void propertyChanged(PropertyState before, PropertyState after) { markPropertyChanged(before.getName()); propertiesModified.add(before); checkAggregates(before.getName()); + propertyUpdated(before, after); } @Override @@ -182,6 +189,7 @@ public void propertyDeleted(PropertyState before) { markPropertyChanged(before.getName()); propertiesModified.add(before); checkAggregates(before.getName()); + propertyUpdated(before, null); } @Override @@ -346,6 +354,36 @@ private void markPropertyChanged(String name) { } } + private void propertyUpdated(PropertyState before, PropertyState after) { + PropertyUpdateCallback callback = context.getPropertyUpdateCallback(); + + //Avoid further work if no callback is present + if (callback == null) { + return; + } + + String propertyName = before != null ? before.getName() : after.getName(); + + if (isIndexable()) { + PropertyDefinition pd = indexingRule.getConfig(propertyName); + if (pd != null) { + callback.propertyUpdated(getPath(), propertyName, pd, before, after); + } + } + + for (Matcher m : matcherState.matched) { + if (m.aggregatesProperty(propertyName)) { + Aggregate.Include i = m.getCurrentInclude(); + if (i instanceof Aggregate.PropertyInclude) { + PropertyDefinition pd = ((Aggregate.PropertyInclude) i).getPropertyDefinition(); + String propertyRelativePath = PathUtils.concat(m.getMatchedPath(), propertyName); + + callback.propertyUpdated(m.getRootPath(), propertyRelativePath, pd, before, after); + } + } + } + } + private IndexDefinition getDefinition() { return context.getDefinition(); } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java index 32dda49ed4..734830a552 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorContext.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Calendar; +import javax.annotation.CheckForNull; import javax.annotation.Nullable; import org.apache.jackrabbit.oak.api.CommitFailedException; @@ -88,6 +89,8 @@ private BinaryTextExtractor textExtractor; + private PropertyUpdateCallback propertyUpdateCallback; + LuceneIndexEditorContext(NodeState root, NodeBuilder definition, @Nullable IndexDefinition indexDefinition, IndexUpdateCallback updateCallback, @@ -127,6 +130,15 @@ public IndexingContext getIndexingContext() { return indexingContext; } + @CheckForNull + public PropertyUpdateCallback getPropertyUpdateCallback() { + return propertyUpdateCallback; + } + + void setPropertyUpdateCallback(PropertyUpdateCallback propertyUpdateCallback) { + this.propertyUpdateCallback = propertyUpdateCallback; + } + /** * close writer if it's not null */ @@ -184,7 +196,7 @@ public long incIndexedNodes() { return indexedNodes; } - private boolean isAsyncIndexing() { + boolean isAsyncIndexing() { return asyncIndexing; } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java index 5ec8a3a6ad..cf46955229 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java @@ -33,6 +33,9 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.IndexingQueue; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexWriterFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneDocumentHolder; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.LuceneIndexPropertyQuery; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyQuery; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; @@ -133,6 +136,9 @@ public Editor getIndexEditor( LuceneIndexWriterFactory writerFactory = null; IndexDefinition indexDefinition = null; boolean asyncIndexing = true; + String indexPath = indexingContext.getIndexPath(); + PropertyIndexUpdateCallback propertyUpdateCallback = null; + if (nrtIndexingEnabled() && !indexingContext.isAsync() && IndexDefinition.supportsSyncOrNRTIndexing(definition)) { //Would not participate in reindexing. Only interested in @@ -153,14 +159,15 @@ public Editor getIndexEditor( //TODO Also check if index has been done once + writerFactory = new LocalIndexWriterFactory(getDocumentHolder(commitContext), - indexingContext.getIndexPath()); + indexPath); //IndexDefinition from tracker might differ from one passed here for reindexing //case which should be fine. However reusing existing definition would avoid //creating definition instance for each commit as this gets executed for each commit if (indexTracker != null){ - indexDefinition = indexTracker.getIndexDefinition(indexingContext.getIndexPath()); + indexDefinition = indexTracker.getIndexDefinition(indexPath); if (indexDefinition != null && !indexDefinition.hasMatchingNodeTypeReg(root)){ log.debug("Detected change in NodeType registry for index {}. Would not use " + "existing index definition", indexDefinition.getIndexPath()); @@ -168,6 +175,19 @@ public Editor getIndexEditor( } } + if (indexDefinition == null) { + indexDefinition = IndexDefinition.newBuilder(root, definition.getNodeState(), + indexPath).build(); + } + + if (indexDefinition.hasSyncPropertyDefinitions()) { + propertyUpdateCallback = new PropertyIndexUpdateCallback(indexPath, definition); + if (indexTracker != null) { + PropertyQuery query = new LuceneIndexPropertyQuery(indexTracker, indexPath); + propertyUpdateCallback.getUniquenessConstraintValidator().setSecondStore(query); + } + } + //Pass on a read only builder to ensure that nothing gets written //at all to NodeStore for local indexing. //TODO [hybrid] This would cause issue with Facets as for faceted fields @@ -184,6 +204,8 @@ public Editor getIndexEditor( LuceneIndexEditorContext context = new LuceneIndexEditorContext(root, definition, indexDefinition, callback, writerFactory, extractedTextCache, augmentorFactory, indexingContext, asyncIndexing); + + context.setPropertyUpdateCallback(propertyUpdateCallback); return new LuceneIndexEditor(context); } return null; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java index f518b13f3d..c89ae3c54e 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map.Entry; import java.util.Set; import java.util.TreeSet; @@ -50,7 +49,7 @@ import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.IndexPathService; import org.apache.jackrabbit.oak.plugins.index.lucene.BadIndexTracker.BadIndexInfo; -import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex.PathStoredFieldVisitor; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Level; import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexConsistencyChecker.Result; diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java index b2251058ee..b1a76a5991 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderService.java @@ -36,6 +36,7 @@ import javax.management.NotCompliantMBeanException; import com.google.common.base.Strings; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.commons.io.FilenameUtils; import org.apache.felix.scr.annotations.Activate; @@ -65,6 +66,7 @@ import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LuceneJournalPropertyService; import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver; @@ -248,6 +250,16 @@ "Cleanup implies purging index blobs marked as deleted earlier during some indexing cycle." ) private static final String PROP_NAME_DELETED_BLOB_COLLECTION_DEFAULT_INTERVAL = "deletedBlobsCollectionInterval"; + + private static final int PROP_INDEX_CLEANER_INTERVAL_DEFAULT = 10*60; + @Property( + intValue = PROP_INDEX_CLEANER_INTERVAL_DEFAULT, + label = "Property Index Cleaner Interval (seconds)", + description = "Cleaner interval time (in seconds) for synchronous property indexes configured as " + + "part of lucene indexes" + ) + private static final String PROP_INDEX_CLEANER_INTERVAL = "propIndexCleanerIntervalInSecs"; + /** * Actively deleted blob must be deleted for at least this long (in seconds) */ @@ -320,6 +332,8 @@ private LuceneIndexEditorProvider editorProvider; + private IndexTracker tracker; + @Activate private void activate(BundleContext bundleContext, Map config) throws NotCompliantMBeanException, IOException { @@ -339,7 +353,7 @@ private void activate(BundleContext bundleContext, Map config) threadPoolSize = PropertiesUtil.toInteger(config.get(PROP_THREAD_POOL_SIZE), PROP_THREAD_POOL_SIZE_DEFAULT); initializeIndexDir(bundleContext, config); initializeExtractedTextCache(bundleContext, config); - IndexTracker tracker = createTracker(bundleContext, config); + tracker = createTracker(bundleContext, config); indexProvider = new LuceneIndexProvider(tracker, scorerFactory, augmentorFactory); initializeActiveBlobCollector(whiteboard, config); initializeLogging(config); @@ -351,6 +365,7 @@ private void activate(BundleContext bundleContext, Map config) registerIndexEditor(bundleContext, tracker, config); registerIndexInfoProvider(bundleContext); registerIndexImporterProvider(bundleContext); + registerPropertyIndexCleaner(config, bundleContext); oakRegs.add(registerMBean(whiteboard, LuceneIndexMBean.class, @@ -481,16 +496,20 @@ private void registerIndexEditor(BundleContext bundleContext, IndexTracker track private IndexTracker createTracker(BundleContext bundleContext, Map config) throws IOException { boolean enableCopyOnRead = PropertiesUtil.toBoolean(config.get(PROP_COPY_ON_READ), true); + IndexTracker tracker; if (enableCopyOnRead){ initializeIndexCopier(bundleContext, config); log.info("Enabling CopyOnRead support. Index files would be copied under {}", indexDir.getAbsolutePath()); if (hybridIndex) { nrtIndexFactory = new NRTIndexFactory(indexCopier, statisticsProvider); } - return new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, indexCopier), nrtIndexFactory); + tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, indexCopier), nrtIndexFactory); + } else { + tracker = new IndexTracker(); } - return new IndexTracker(); + tracker.setAsyncIndexInfoService(asyncIndexInfoService); + return tracker; } private void initializeIndexCopier(BundleContext bundleContext, Map config) throws IOException { @@ -745,6 +764,24 @@ private long getSafeTimestampForDeletedBlobs(CheckpointMBean checkpointMBean) { return timestamp; } + + private void registerPropertyIndexCleaner(Map config, BundleContext bundleContext) { + int cleanerInterval = PropertiesUtil.toInteger(config.get(PROP_INDEX_CLEANER_INTERVAL), + PROP_INDEX_CLEANER_INTERVAL_DEFAULT); + + if (cleanerInterval <= 0) { + log.info("Property index cleaner would not be registered"); + return; + } + + PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService); + oakRegs.add(scheduleWithFixedDelay(whiteboard, cleaner, + ImmutableMap.of("scheduler.name", PropertyIndexCleaner.class.getName()), + cleanerInterval, true, true)); + log.info("Property index cleaner configured to run every [{}] seconds", cleanerInterval); + } + + protected void bindNodeAggregator(QueryIndex.NodeAggregator aggregator) { this.nodeAggregator = aggregator; initialize(); diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java index 7e7f208a63..2c17fa2bd8 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java @@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.AbstractIterator; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Queues; import com.google.common.collect.Sets; @@ -47,10 +48,13 @@ import org.apache.jackrabbit.oak.commons.json.JsopWriter; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexPlanner.PlanResult; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexPlanner.PropertyIndexResult; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup; import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.plugins.index.lucene.spi.FulltextQueryTermsProvider; import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper; import org.apache.jackrabbit.oak.plugins.index.lucene.util.MoreLikeThisHelper; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SpellcheckHelper; import org.apache.jackrabbit.oak.plugins.index.lucene.util.SuggestHelper; import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; @@ -71,6 +75,7 @@ import org.apache.jackrabbit.oak.spi.query.QueryLimits; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.commons.benchmark.PerfLogger; +import org.apache.jackrabbit.oak.spi.state.NodeStateUtils; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; @@ -78,12 +83,10 @@ import org.apache.lucene.facet.Facets; import org.apache.lucene.facet.LabelAndValue; import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.StoredFieldVisitor; import org.apache.lucene.index.Term; import org.apache.lucene.queries.CustomScoreQuery; import org.apache.lucene.queryparser.classic.ParseException; @@ -130,7 +133,6 @@ import static org.apache.jackrabbit.oak.commons.PathUtils.denotesRoot; import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.ANALYZED_FIELD_PREFIX; -import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.NATIVE_SORT_ORDER; import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.VERSION; import static org.apache.jackrabbit.oak.plugins.index.lucene.TermFactory.newAncestorTerm; @@ -578,6 +580,11 @@ public long getSize() { return -1; } }; + + if (pr.hasPropertyIndexResult()) { + itr = mergePropertyIndexResult(plan, rootState, itr); + } + return new LucenePathCursor(itr, plan, settings, sizeEstimator); } @@ -1529,6 +1536,20 @@ private static Query newDepthQuery(String path) { return NumericRangeQuery.newIntRange(FieldNames.PATH_DEPTH, depth, depth, true, true); } + private static Iterator mergePropertyIndexResult(IndexPlan plan, NodeState rootState, + Iterator itr) { + PlanResult pr = getPlanResult(plan); + HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(pr.indexPath, + NodeStateUtils.getNode(rootState, pr.indexPath)); + PropertyIndexResult pir = pr.getPropertyIndexResult(); + Iterable paths = lookup.query(plan.getFilter(), pir.pd, pir.propertyName, pir.pr); + Iterator propIndexItr = Iterators.transform(paths.iterator(), + (path) -> new LuceneResultRow(path, 0, null, null, null)); + + //Property index itr should come first + return Iterators.concat(propIndexItr, itr); + } + static class LuceneResultRow { final String path; final double score; @@ -1697,31 +1718,4 @@ public long getSize(SizePrecision precision, long max) { } } - static class PathStoredFieldVisitor extends StoredFieldVisitor { - - private String path; - private boolean pathVisited; - - @Override - public Status needsField(FieldInfo fieldInfo) throws IOException { - if (PATH.equals(fieldInfo.name)) { - return Status.YES; - } - return pathVisited ? Status.STOP : Status.NO; - } - - @Override - public void stringField(FieldInfo fieldInfo, String value) - throws IOException { - if (PATH.equals(fieldInfo.name)) { - path = value; - pathVisited = true; - } - } - - public String getPath() { - return path; - } - } - } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java index 93ae9ef45e..f3a5365925 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyDefinition.java @@ -42,7 +42,7 @@ import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.PROP_WEIGHT; import static org.apache.jackrabbit.oak.plugins.index.lucene.util.ConfigUtil.getOptionalValue; -class PropertyDefinition { +public class PropertyDefinition { private static final Logger log = LoggerFactory.getLogger(PropertyDefinition.class); /** * The default boost: 1.0f. @@ -106,7 +106,7 @@ /** * For function-based indexes: the function name, in Polish notation. - */ + */ final String function; /** @@ -114,7 +114,11 @@ */ final String[] functionCode; - final ValuePattern valuePattern; + public final ValuePattern valuePattern; + + public final boolean sync; + + public final boolean unique; public PropertyDefinition(IndexingRule idxDefn, String nodeName, NodeState defn) { this.isRegexp = getOptionalValue(defn, PROP_IS_REGEX, false); @@ -135,8 +139,6 @@ public PropertyDefinition(IndexingRule idxDefn, String nodeName, NodeState defn) this.analyzed = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_ANALYZED, false); } - //If node is not set for full text then a property definition indicates that definition is for property index - this.propertyIndex = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_PROPERTY_INDEX, false); this.ordered = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_ORDERED, false); this.includedPropertyTypes = IndexDefinition.getSupportedTypes(defn, LuceneIndexConstants.PROP_INCLUDED_TYPE, IndexDefinition.TYPES_ALLOW_ALL); @@ -156,6 +158,11 @@ public PropertyDefinition(IndexingRule idxDefn, String nodeName, NodeState defn) getOptionalValue(defn, LuceneIndexConstants.PROP_FUNCTION, null)); this.functionCode = FunctionIndexProcessor.getFunctionCode(this.function); this.valuePattern = new ValuePattern(defn); + this.unique = getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_UNIQUE, false); + this.sync = unique || getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_SYNC, false); + + //If some property is set to sync then propertyIndex mode is always enabled + this.propertyIndex = sync || getOptionalValueIfIndexed(defn, LuceneIndexConstants.PROP_PROPERTY_INDEX, false); validate(); } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyUpdateCallback.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyUpdateCallback.java new file mode 100644 index 0000000000..fbad6b6f00 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/PropertyUpdateCallback.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import javax.annotation.Nullable; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; + +/** + * Callback to be invoked for each indexable property change + */ +public interface PropertyUpdateCallback { + + /** + * Invoked upon any change in property either added, updated or removed. + * Implementation can determine if property is added, updated or removed based + * on whether before or after is null + * + * @param nodePath path of node for which is to be indexed for this property change + * @param propertyRelativePath relative path of the property wrt the indexed node + * @param pd property definition associated with the property to be indexed + * @param before before state. Is null when property is added. For other cases its not null + * @param after after state of the property. Is null when property is removed. For other cases its not null + */ + void propertyUpdated(String nodePath, String propertyRelativePath, PropertyDefinition pd, + @Nullable PropertyState before, @Nullable PropertyState after); + + /** + * Invoked after editor has traversed all the changes + * + * @throws CommitFailedException in case some validation fails + */ + void done() throws CommitFailedException; + +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcher.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcher.java new file mode 100644 index 0000000000..fed89a3811 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcher.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.Objects; + +import com.google.common.collect.Iterables; +import com.google.common.primitives.Ints; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; + +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_HEAD_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_PREVIOUS_BUCKET; + +/** + * Takes care of switching the buckets used by non unique property indexes + * based on the associated async indexer lastIndexedTo time. + * + * It also ensures that unnecessary changes are not done if the property index + * does not get updated + */ +class BucketSwitcher { + private final NodeBuilder builder; + + public BucketSwitcher(NodeBuilder builder) { + this.builder = builder; + } + + public boolean switchBucket(long lastIndexedTo) { + String head = builder.getString(PROP_HEAD_BUCKET); + + if (head == null) { + //Cleaner ran before any updates to index + //nothing to do further + return false; + } + + NodeBuilder headb = builder.getChildNode(head); + long headLastIndexedTo = getOptionalValue(headb, PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH, 0); + + if (headLastIndexedTo > lastIndexedTo) { + //> Should not happen in general as it means that + //async indexer clock switched back for some reason + return false; + } + + if (headLastIndexedTo == lastIndexedTo) { + //Async indexer has yet not moved so keep current state + return false; + } + + if (asyncIndexedToTimeSameAsPrevious(lastIndexedTo)) { + //Async indexer has yet not moved so keep current state + return false; + } + + if (headb.getChildNodeCount(1) > 0) { + //Bucket non empty case + //Create new head bucket and switch previous to current head + String nextHeadName = String.valueOf(Integer.parseInt(head) + 1); + builder.child(nextHeadName); + builder.setProperty(PROP_HEAD_BUCKET, nextHeadName); + builder.setProperty(PROP_PREVIOUS_BUCKET, head); + headb.setProperty(PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH, lastIndexedTo); + } else { + //Bucket remains empty + //Avoid unnecessary new bucket creation or any other changes + if (headLastIndexedTo == 0) { + //Only update time if not already set + headb.setProperty(PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH, lastIndexedTo); + } + //Remove any previous bucket reference + builder.removeProperty(PROP_PREVIOUS_BUCKET); + } + + return builder.isModified(); + } + + public Iterable getOldBuckets() { + String head = builder.getString(PROP_HEAD_BUCKET); + String previous = builder.getString(PROP_PREVIOUS_BUCKET); + return Iterables.filter(builder.getChildNodeNames(), + name -> !Objects.equals(name, head) && !Objects.equals(name, previous) + ); + } + + private boolean asyncIndexedToTimeSameAsPrevious(long lastIndexedTo) { + String previous = builder.getString(PROP_PREVIOUS_BUCKET); + if (previous != null) { + long previousAsyncIndexedTo = getOptionalValue(builder.getChildNode(previous), + PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH, 0); + return previousAsyncIndexedTo == lastIndexedTo; + } + return false; + } + + private static long getOptionalValue(NodeBuilder nb, String propName, int defaultVal){ + PropertyState ps = nb.getProperty(propName); + return ps == null ? defaultVal : ps.getValue(Type.LONG); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookup.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookup.java new file mode 100644 index 0000000000..c8a988f419 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookup.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.Collections; +import java.util.Set; + +import com.google.common.collect.Iterables; +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexUtil; +import org.apache.jackrabbit.oak.plugins.index.property.ValuePatternUtil; +import org.apache.jackrabbit.oak.plugins.index.property.strategy.ContentMirrorStoreStrategy; +import org.apache.jackrabbit.oak.plugins.index.property.strategy.UniqueEntryStoreStrategy; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import static com.google.common.collect.Iterables.transform; +import static org.apache.jackrabbit.oak.commons.PathUtils.isAbsolute; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_HEAD_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_PREVIOUS_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexUtil.encode; + +public class HybridPropertyIndexLookup { + private final String indexPath; + private final NodeState indexState; + + public HybridPropertyIndexLookup(String indexPath, NodeState indexState) { + this.indexPath = indexPath; + this.indexState = indexState; + } + + /** + * Performs query based on provided property restriction + * + * @param filter filter from the query being performed + * @param pd property definition as per index definition + * @param propertyName actual property name which may or may not be same as + * property name in property restriction + * @param restriction property restriction matching given property + * @return iterable consisting of absolute paths as per index content + */ + public Iterable query(Filter filter, PropertyDefinition pd, + String propertyName, Filter.PropertyRestriction restriction) { + //The propertyName may differ from name in restriction. For e.g. for relative properties + //the restriction property name can be 'jcr:content/status' while the index has indexed + //for 'status' + + Set values = ValuePatternUtil.getAllValues(restriction); + Set encodedValues = PropertyIndexUtil.encode(values); + return query(filter, pd, propertyName, encodedValues); + } + + public Iterable query(Filter filter, PropertyDefinition pd, + String propertyName, PropertyValue value) { + return query(filter, pd, propertyName, encode(value, pd.valuePattern)); + } + + private Iterable query(Filter filter, PropertyDefinition pd, + String propertyName, Set encodedValues) { + String propIdxNodeName = HybridPropertyIndexUtil.getNodeName(propertyName); + NodeState propIndexRootNode = indexState.getChildNode(PROPERTY_INDEX); + NodeState propIndexNode = propIndexRootNode.getChildNode(propIdxNodeName); + if (!propIndexNode.exists()) { + return Collections.emptyList(); + } + + //TODO Check for non root indexes + String indexName = indexPath + "(" + propertyName + ")"; + Iterable result; + if (pd.unique) { + result = queryUnique(filter, indexName, propIndexRootNode, propIdxNodeName, encodedValues); + } else { + result = querySimple(filter, indexName, propIndexNode, encodedValues); + } + + return transform(result, path -> isAbsolute(path) ? path : "/" + path); + } + + private static Iterable queryUnique(Filter filter, String indexName, NodeState propIndexRootNode, + String propIdxNodeName, Set values) { + UniqueEntryStoreStrategy s = new UniqueEntryStoreStrategy(propIdxNodeName); + return s.query(filter, indexName, propIndexRootNode, values); + } + + private static Iterable querySimple(Filter filter, String indexName, NodeState propIndexNode, + Set values) { + return Iterables.concat( + queryBucket(filter, indexName, propIndexNode, PROP_HEAD_BUCKET, values), + queryBucket(filter, indexName, propIndexNode, PROP_PREVIOUS_BUCKET, values) + ); + } + + private static Iterable queryBucket(Filter filter, String indexName, NodeState propIndexNode, + String bucketPropName, Set values) { + String bucketName = propIndexNode.getString(bucketPropName); + if (bucketName == null) { + return Collections.emptyList(); + } + ContentMirrorStoreStrategy s = new ContentMirrorStoreStrategy(bucketName); + return s.query(filter, indexName, propIndexNode, bucketName, values); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexUtil.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexUtil.java new file mode 100644 index 0000000000..4b6d94c778 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexUtil.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import org.apache.jackrabbit.JcrConstants; + +class HybridPropertyIndexUtil { + /** + * Node name under which all property indexes are created + */ + static final String PROPERTY_INDEX = ":property-index"; + + /** + * Property name referring to 'head' bucket + */ + static final String PROP_HEAD_BUCKET = "head"; + + /** + * Property name referring to 'previous' bucket + */ + static final String PROP_PREVIOUS_BUCKET = "previous"; + + /** + * Property set on each bucket to record that it entries after + * given async indexer state i.e. lastIndexTo time for the associated + * asyn indexer + */ + static final String PROP_ASYNC_INDEXED_TO_TIME_AT_SWITCH = "asyncIndexedToTimeAtSwitch"; + + /** + * Creation time used for entries in unique property index. Instead of + * storing the data as calendar it stores it as epoch time + */ + static final String PROP_CREATED = JcrConstants.JCR_CREATED; + + static final String PROP_STORAGE_TYPE = "storageType"; + + static final String STORAGE_TYPE_CONTENT_MIRROR = "contentMirror"; + + static final String STORAGE_TYPE_UNIQUE = "unique"; + + static String getNodeName(String propertyRelativePath) { + return propertyRelativePath.replace('/', '_'); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQuery.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQuery.java new file mode 100644 index 0000000000..765e72c9fb --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQuery.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexNode; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.PathStoredFieldVisitor; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Performs simple property=value query against a Lucene index + */ +public class LuceneIndexPropertyQuery implements PropertyQuery { + private static final Logger log = LoggerFactory.getLogger(LuceneIndexPropertyQuery.class); + private final IndexTracker tracker; + private final String indexPath; + + public LuceneIndexPropertyQuery(IndexTracker tracker, String indexPath) { + this.tracker = tracker; + this.indexPath = indexPath; + } + + @Override + public Iterable getIndexedPaths(String propertyRelativePath, String value) { + List indexPaths = new ArrayList<>(2); + IndexNode indexNode = tracker.acquireIndexNode(indexPath); + if (indexNode != null) { + try { + TermQuery query = new TermQuery(new Term(propertyRelativePath, value)); + //By design such query should not result in more than 1 result. + //So just use 10 as batch size + TopDocs docs = indexNode.getSearcher().search(query, 10); + + IndexReader reader = indexNode.getSearcher().getIndexReader(); + for (ScoreDoc d : docs.scoreDocs) { + PathStoredFieldVisitor visitor = new PathStoredFieldVisitor(); + reader.document(d.doc, visitor); + indexPaths.add(visitor.getPath()); + } + } catch (IOException e) { + log.warn("Error occurred while checking index {} for unique value " + + "[{}] for [{}]", indexPath,value, propertyRelativePath, e); + } finally { + indexNode.release(); + } + } + return indexPaths; + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java new file mode 100644 index 0000000000..57370011c2 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleaner.java @@ -0,0 +1,273 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.base.Stopwatch; +import com.google.common.collect.ImmutableMap; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; +import org.apache.jackrabbit.oak.plugins.index.IndexPathService; +import org.apache.jackrabbit.oak.plugins.index.IndexUtils; +import org.apache.jackrabbit.oak.spi.commit.CommitContext; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.commit.SimpleCommitContext; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_STORAGE_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_CONTENT_MIRROR; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_UNIQUE; +import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode; + +public class PropertyIndexCleaner implements Runnable{ + private final Logger log = LoggerFactory.getLogger(getClass()); + private final NodeStore nodeStore; + private final IndexPathService indexPathService; + private final AsyncIndexInfoService asyncIndexInfoService; + private UniqueIndexCleaner uniqueIndexCleaner = new UniqueIndexCleaner(TimeUnit.HOURS, 1); + private Map lastAsyncInfo = Collections.emptyMap(); + + public PropertyIndexCleaner(NodeStore nodeStore, IndexPathService indexPathService, + AsyncIndexInfoService asyncIndexInfoService) { + this.nodeStore = checkNotNull(nodeStore); + this.indexPathService = checkNotNull(indexPathService); + this.asyncIndexInfoService = checkNotNull(asyncIndexInfoService); + } + + @Override + public void run() { + try{ + performCleanup(); + } catch (Exception e) { + log.warn("Cleanup run failed with error", e); + } + } + + /** + * Performs the cleanup run + * + * @return true if the cleanup was attempted + */ + public boolean performCleanup() throws CommitFailedException { + Stopwatch w = Stopwatch.createStarted(); + Map asyncInfo = asyncIndexInfoService.getIndexedUptoPerLane(); + if (lastAsyncInfo.equals(asyncInfo)) { + log.debug("No change found in async state from last run {}. Skipping the run", asyncInfo); + return false; + } + CleanupStats stats = new CleanupStats(); + List syncIndexes = getSyncIndexPaths(); + IndexInfo indexInfo = switchBucketsAndCollectIndexData(syncIndexes, asyncInfo, stats); + + purgeOldBuckets(indexInfo.oldBucketPaths, stats); + purgeOldUniqueIndexEntries(indexInfo.uniqueIndexPaths, stats); + lastAsyncInfo = asyncInfo; + + if (w.elapsed(TimeUnit.MINUTES) > 5) { + log.info("Property index cleanup done in {}. {}", w, stats); + } else { + log.debug("Property index cleanup done in {}. {}", w, stats); + } + + return true; + } + + /** + * Specifies the threshold for created time such that only those entries + * in unique indexes are purged which have + * + * async indexer time - creation time > threshold + * + * @param unit time unit + * @param time time value in given unit + */ + public void setCreatedTimeThreshold(TimeUnit unit, long time) { + uniqueIndexCleaner = new UniqueIndexCleaner(unit, time); + } + + List getSyncIndexPaths() { + List indexPaths = new ArrayList<>(); + NodeState root = nodeStore.getRoot(); + for (String indexPath : indexPathService.getIndexPaths()) { + NodeState idx = getNode(root, indexPath); + if (TYPE_LUCENE.equals(idx.getString(TYPE_PROPERTY_NAME)) + && idx.hasChildNode(PROPERTY_INDEX)) { + indexPaths.add(indexPath); + } + } + return indexPaths; + } + + private IndexInfo switchBucketsAndCollectIndexData(List indexPaths, + Map asyncInfo, CleanupStats stats) + throws CommitFailedException { + IndexInfo indexInfo = new IndexInfo(); + NodeState root = nodeStore.getRoot(); + NodeBuilder builder = root.builder(); + + boolean modified = false; + for (String indexPath : indexPaths) { + NodeState idx = getNode(root, indexPath); + NodeBuilder idxb = child(builder, indexPath); + String laneName = IndexUtils.getAsyncLaneName(idx, indexPath); + Long lastIndexedTo = asyncInfo.get(laneName); + + if (lastIndexedTo == null) { + log.warn("Not able to determine async index info for lane {}. " + + "Known lanes {}", laneName, asyncInfo.keySet()); + continue; + } + + NodeState propertyIndexNode = idx.getChildNode(PROPERTY_INDEX); + NodeBuilder propIndexNodeBuilder = idxb.getChildNode(PROPERTY_INDEX); + + for (ChildNodeEntry cne : propertyIndexNode.getChildNodeEntries()) { + NodeState propIdxState = cne.getNodeState(); + String propName = cne.getName(); + if (simplePropertyIndex(propIdxState)) { + + NodeBuilder propIdx = propIndexNodeBuilder.getChildNode(propName); + BucketSwitcher bs = new BucketSwitcher(propIdx); + + modified |= bs.switchBucket(lastIndexedTo); + + for (String bucketName : bs.getOldBuckets()) { + String bucketPath = PathUtils.concat(indexPath, PROPERTY_INDEX, propName, bucketName); + indexInfo.oldBucketPaths.add(bucketPath); + stats.purgedIndexPaths.add(indexPath); + } + } else if (uniquePropertyIndex(propIdxState)) { + String indexNodePath = PathUtils.concat(indexPath, PROPERTY_INDEX, propName); + indexInfo.uniqueIndexPaths.put(indexNodePath, lastIndexedTo); + } + } + } + + if (modified) { + merge(builder); + + } + return indexInfo; + } + + private void purgeOldBuckets(List bucketPaths, CleanupStats stats) throws CommitFailedException { + if (bucketPaths.isEmpty()) { + return; + } + + NodeState root = nodeStore.getRoot(); + NodeBuilder builder = root.builder(); + + for (String path : bucketPaths) { + NodeBuilder bucket = child(builder, path); + //TODO Recursive delete to avoid large transaction + bucket.remove(); + } + + stats.bucketCount = bucketPaths.size(); + merge(builder); + } + + private void purgeOldUniqueIndexEntries(Map asyncInfo, CleanupStats stats) throws CommitFailedException { + NodeState root = nodeStore.getRoot(); + NodeBuilder builder = root.builder(); + + for (Map.Entry e : asyncInfo.entrySet()) { + String indexNodePath = e.getKey(); + NodeBuilder idxb = child(builder, indexNodePath); + int removalCount = uniqueIndexCleaner.clean(idxb, e.getValue()); + if (removalCount > 0) { + stats.purgedIndexPaths.add(PathUtils.getAncestorPath(indexNodePath, 2)); + log.debug("Removed [{}] entries from [{}]", removalCount, indexNodePath); + } + stats.uniqueIndexEntryRemovalCount += removalCount; + } + + if (stats.uniqueIndexEntryRemovalCount > 0) { + merge(builder); + } + } + + private void merge(NodeBuilder builder) throws CommitFailedException { + //TODO Configure conflict hooks + //TODO Configure validator + nodeStore.merge(builder, EmptyHook.INSTANCE, createCommitInfo()); + } + + private static CommitInfo createCommitInfo() { + Map info = ImmutableMap.of(CommitContext.NAME, new SimpleCommitContext()); + return new CommitInfo(CommitInfo.OAK_UNKNOWN, CommitInfo.OAK_UNKNOWN, info); + } + + private static NodeBuilder child(NodeBuilder nb, String path) { + for (String name : PathUtils.elements(checkNotNull(path))) { + //Use getChildNode to avoid creating new entries by default + nb = nb.getChildNode(name); + } + return nb; + } + + private static boolean simplePropertyIndex(NodeState propIdxState) { + return STORAGE_TYPE_CONTENT_MIRROR.equals(propIdxState.getString(PROP_STORAGE_TYPE)); + } + + private static boolean uniquePropertyIndex(NodeState propIdxState) { + return STORAGE_TYPE_UNIQUE.equals(propIdxState.getString(PROP_STORAGE_TYPE)); + } + + private static final class IndexInfo { + final List oldBucketPaths = new ArrayList<>(); + + /* indexPath, lastIndexedTo */ + final Map uniqueIndexPaths = new HashMap<>(); + } + + private static class CleanupStats { + int uniqueIndexEntryRemovalCount; + int bucketCount; + Set purgedIndexPaths = new HashSet<>(); + + @Override + public String toString() { + return String.format("Removed %d index buckets, %d unique index entries " + + "from indexes %s", bucketCount, uniqueIndexEntryRemovalCount, purgedIndexPaths); + } + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexQuery.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexQuery.java new file mode 100644 index 0000000000..cee2f102d2 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexQuery.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; + +import static com.google.common.base.Preconditions.checkState; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_STORAGE_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_UNIQUE; + +/** + * Performs simple property=value query against a unique property index storage + */ +class PropertyIndexQuery implements PropertyQuery { + private final NodeBuilder builder; + + public PropertyIndexQuery(NodeBuilder builder) { + this.builder = builder; + } + + @Override + public Iterable getIndexedPaths(String propertyRelativePath, String value) { + NodeBuilder idxb = getIndexNode(propertyRelativePath); + checkState(STORAGE_TYPE_UNIQUE.equals(idxb.getString(PROP_STORAGE_TYPE))); + + NodeBuilder entry = idxb.child(value); + return entry.getProperty("entry").getValue(Type.STRINGS); + } + + private NodeBuilder getIndexNode(String propertyRelativePath) { + NodeBuilder propertyIndex = builder.child(PROPERTY_INDEX); + String nodeName = HybridPropertyIndexUtil.getNodeName(propertyRelativePath); + return propertyIndex.child(nodeName); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexUpdateCallback.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexUpdateCallback.java new file mode 100644 index 0000000000..c21a1fd5f3 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexUpdateCallback.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.HashSet; +import java.util.Set; + +import javax.annotation.Nullable; +import javax.jcr.PropertyType; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.property.ValuePattern; +import org.apache.jackrabbit.oak.plugins.index.property.strategy.ContentMirrorStoreStrategy; +import org.apache.jackrabbit.oak.plugins.index.property.strategy.UniqueEntryStoreStrategy; +import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.stats.Clock; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Suppliers.ofInstance; +import static com.google.common.collect.Sets.newHashSet; +import static java.util.Collections.emptySet; +import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_CONTENT_NODE_NAME; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_CREATED; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_HEAD_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_STORAGE_TYPE; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_CONTENT_MIRROR; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.STORAGE_TYPE_UNIQUE; +import static org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexUtil.encode; + +public class PropertyIndexUpdateCallback implements PropertyUpdateCallback { + private static final Logger log = LoggerFactory.getLogger(PropertyIndexUpdateCallback.class); + private static final String DEFAULT_HEAD_BUCKET = String.valueOf(1); + + private final NodeBuilder builder; + private final String indexPath; + private final UniquenessConstraintValidator uniquenessConstraintValidator; + private final long updateTime; + + public PropertyIndexUpdateCallback(String indexPath, NodeBuilder builder) { + this(indexPath, builder, Clock.SIMPLE); + } + + public PropertyIndexUpdateCallback(String indexPath, NodeBuilder builder, Clock clock) { + this.builder = builder; + this.indexPath = indexPath; + this.updateTime = clock.getTime(); + this.uniquenessConstraintValidator = new UniquenessConstraintValidator(indexPath, builder); + } + + @Override + public void propertyUpdated(String nodePath, String propertyRelativePath, PropertyDefinition pd, + @Nullable PropertyState before, @Nullable PropertyState after) { + if (!pd.sync) { + return; + } + + Set beforeKeys = getValueKeys(before, pd.valuePattern); + Set afterKeys = getValueKeys(after, pd.valuePattern); + + //Remove duplicates + Set sharedKeys = newHashSet(beforeKeys); + sharedKeys.retainAll(afterKeys); + beforeKeys.removeAll(sharedKeys); + afterKeys.removeAll(sharedKeys); + + if (!beforeKeys.isEmpty() || !afterKeys.isEmpty()){ + NodeBuilder indexNode = getIndexNode(propertyRelativePath, pd.unique); + + if (pd.unique) { + UniqueEntryStoreStrategy s = new UniqueEntryStoreStrategy(INDEX_CONTENT_NODE_NAME, + (nb) -> nb.setProperty(PROP_CREATED, updateTime)); + s.update(ofInstance(indexNode), + nodePath, + null, + null, + beforeKeys, + afterKeys); + uniquenessConstraintValidator.add(propertyRelativePath, afterKeys); + } else { + ContentMirrorStoreStrategy s = new ContentMirrorStoreStrategy(); + s.update(ofInstance(indexNode), + nodePath, + null, + null, + emptySet(), //Disable pruning with empty before keys + afterKeys); + } + + if (log.isTraceEnabled()) { + log.trace("[{}] Property index updated for [{}/@{}] with values {}", indexPath, nodePath, + propertyRelativePath, afterKeys); + } + } + } + + @Override + public void done() throws CommitFailedException { + uniquenessConstraintValidator.validate(); + } + + public UniquenessConstraintValidator getUniquenessConstraintValidator() { + return uniquenessConstraintValidator; + } + + private NodeBuilder getIndexNode(String propertyRelativePath, boolean unique) { + NodeBuilder propertyIndex = builder.child(PROPERTY_INDEX); + + String nodeName = HybridPropertyIndexUtil.getNodeName(propertyRelativePath); + if (unique) { + return getUniqueIndexBuilder(propertyIndex, nodeName); + } else { + return getSimpleIndexBuilder(propertyIndex, nodeName); + } + } + + private NodeBuilder getSimpleIndexBuilder(NodeBuilder propertyIndex, String nodeName) { + NodeBuilder idx = propertyIndex.child(nodeName); + if (idx.isNew()) { + idx.setProperty(PROP_HEAD_BUCKET, DEFAULT_HEAD_BUCKET); + idx.setProperty(PROP_STORAGE_TYPE, STORAGE_TYPE_CONTENT_MIRROR); + } + + String headBucketName = idx.getString(PROP_HEAD_BUCKET); + checkNotNull(headBucketName, "[%s] property not found in [%s] for index [%s]", + PROP_HEAD_BUCKET, idx, indexPath); + + return idx.child(headBucketName); + } + + private static NodeBuilder getUniqueIndexBuilder(NodeBuilder propertyIndex, String nodeName) { + NodeBuilder idx = propertyIndex.child(nodeName); + if (idx.isNew()) { + idx.setProperty(PROP_STORAGE_TYPE, STORAGE_TYPE_UNIQUE); + } + return idx; + } + + private static Set getValueKeys(PropertyState property, ValuePattern pattern) { + Set keys = new HashSet<>(); + if (property != null + && property.getType().tag() != PropertyType.BINARY + && property.count() != 0) { + keys.addAll(encode(PropertyValues.create(property), pattern)); + } + return keys; + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyQuery.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyQuery.java new file mode 100644 index 0000000000..8d178279ef --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyQuery.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.Collections; + +/** + * Abstraction to enable performing simple property=value query + * across various types of storage + */ +public interface PropertyQuery { + PropertyQuery DEFAULT = (propertyRelativePath, value) -> Collections.emptyList(); + + Iterable getIndexedPaths(String propertyRelativePath, String value); +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleaner.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleaner.java new file mode 100644 index 0000000000..f545055af7 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleaner.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; + +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_CREATED; + +class UniqueIndexCleaner { + private final long createTimeMarginMillis; + + public UniqueIndexCleaner(TimeUnit timeUnit, long createTimeMargin) { + this.createTimeMarginMillis = timeUnit.toMillis(createTimeMargin); + } + + public int clean(NodeBuilder builder, long lastIndexedTo) { + int removalCount = 0; + NodeState baseState = builder.getBaseState(); + for (ChildNodeEntry e : baseState.getChildNodeEntries()) { + long entryCreationTime = e.getNodeState().getLong(PROP_CREATED); + if (entryCovered(entryCreationTime, lastIndexedTo)) { + builder.child(e.getName()).remove(); + removalCount++; + } + } + return removalCount; + } + + private boolean entryCovered(long entryCreationTime, long lastIndexedTo) { + //Would be safer to add some margin as entryCreationTime as recorded + //is not same as actual commit time + return (lastIndexedTo - entryCreationTime) >= createTimeMarginMillis; + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidator.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidator.java new file mode 100644 index 0000000000..8b6b216935 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidator.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.Map; +import java.util.Set; + +import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.collect.Multimap; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; + +import static com.google.common.base.Preconditions.checkNotNull; +import static org.apache.jackrabbit.oak.api.CommitFailedException.CONSTRAINT; + +/** + * Performs validation related to unique index by ensuring that for + * given property value only one indexed entry is present. The query + * is performed against multiple stores + * + * - Property storage - Stores the recently added unique keys via UniqueStore strategy + * - Lucene storage - Stores the long term index in lucene + */ +public class UniquenessConstraintValidator { + private final String indexPath; + private final Multimap uniqueKeys = HashMultimap.create(); + private final PropertyQuery firstStore; + private PropertyQuery secondStore = PropertyQuery.DEFAULT; + + public UniquenessConstraintValidator(String indexPath, NodeBuilder builder) { + this.indexPath = indexPath; + this.firstStore = new PropertyIndexQuery(builder); + } + + public void add(String propertyRelativePath, Set afterKeys) { + uniqueKeys.putAll(propertyRelativePath, afterKeys); + } + + public void validate() throws CommitFailedException { + for (Map.Entry e : uniqueKeys.entries()) { + String propertyRelativePath = e.getKey(); + Iterable indexedPaths = getIndexedPaths(propertyRelativePath, e.getValue()); + Set allPaths = ImmutableSet.copyOf(indexedPaths); + if (allPaths.size() > 1) { + String msg = String.format("Uniqueness constraint violated for property [%s] for " + + "index [%s]. IndexedPaths %s", propertyRelativePath, indexPath, allPaths); + throw new CommitFailedException(CONSTRAINT, 30, msg); + } + } + } + + public void setSecondStore(PropertyQuery secondStore) { + this.secondStore = checkNotNull(secondStore); + } + + private Iterable getIndexedPaths(String propertyRelativePath, String value) { + return Iterables.concat( + firstStore.getIndexedPaths(propertyRelativePath, value), + secondStore.getIndexedPaths(propertyRelativePath, value) + ); + } +} diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java index cba5ee222a..2438b24ddc 100644 --- a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/IndexDefinitionBuilder.java @@ -369,6 +369,16 @@ public PropertyRule valueIncludedPrefixes(String... values){ return this; } + public PropertyRule sync(){ + propTree.setProperty(LuceneIndexConstants.PROP_SYNC, true); + return this; + } + + public PropertyRule unique(){ + propTree.setProperty(LuceneIndexConstants.PROP_UNIQUE, true); + return this; + } + public IndexRule enclosingRule(){ return indexRule; } diff --git a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/PathStoredFieldVisitor.java b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/PathStoredFieldVisitor.java new file mode 100644 index 0000000000..7e156a0101 --- /dev/null +++ b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/util/PathStoredFieldVisitor.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.util; + +import java.io.IOException; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.StoredFieldVisitor; + +import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldNames.PATH; + +public class PathStoredFieldVisitor extends StoredFieldVisitor { + + private String path; + private boolean pathVisited; + + @Override + public Status needsField(FieldInfo fieldInfo) throws IOException { + if (PATH.equals(fieldInfo.name)) { + return Status.YES; + } + return pathVisited ? Status.STOP : Status.NO; + } + + @Override + public void stringField(FieldInfo fieldInfo, String value) + throws IOException { + if (PATH.equals(fieldInfo.name)) { + path = value; + pathVisited = true; + } + } + + public String getPath() { + return path; + } +} diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java index dbef781780..d1641e1ff2 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinitionTest.java @@ -29,6 +29,7 @@ import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.IndexingRule; import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; import org.apache.jackrabbit.oak.plugins.index.lucene.util.TokenizerChain; import org.apache.jackrabbit.oak.plugins.index.lucene.writer.CommitMitigatingTieredMergePolicy; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; @@ -82,6 +83,7 @@ public void defaultConfig() throws Exception{ IndexDefinition idxDefn = new IndexDefinition(root, builder.getNodeState(), "/foo"); assertTrue(idxDefn.saveDirListing()); assertFalse(idxDefn.isNRTIndexingEnabled()); + assertFalse(idxDefn.hasSyncPropertyDefinitions()); } @Test @@ -975,6 +977,36 @@ public void nodeTypeChange() throws Exception{ assertTrue(defn.hasMatchingNodeTypeReg(root3)); } + @Test + public void uniqueIsSync() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").unique(); + + IndexDefinition defn = IndexDefinition.newBuilder(root, defnb.build(), "/foo").build(); + assertTrue(defn.getApplicableIndexingRule("nt:base").getConfig("foo").sync); + assertTrue(defn.getApplicableIndexingRule("nt:base").getConfig("foo").unique); + assertTrue(defn.getApplicableIndexingRule("nt:base").getConfig("foo").propertyIndex); + } + + @Test + public void syncIsProperty() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").sync(); + + IndexDefinition defn = IndexDefinition.newBuilder(root, defnb.build(), "/foo").build(); + assertTrue(defn.getApplicableIndexingRule("nt:base").getConfig("foo").sync); + assertTrue(defn.getApplicableIndexingRule("nt:base").getConfig("foo").propertyIndex); + } + + @Test + public void syncPropertyDefinitions() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").sync(); + + IndexDefinition defn = IndexDefinition.newBuilder(root, defnb.build(), "/foo").build(); + assertTrue(defn.hasSyncPropertyDefinitions()); + } + //TODO indexesAllNodesOfMatchingType - with nullCheckEnabled private static IndexingRule getRule(IndexDefinition defn, String typeName){ diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java index af12bcd5b2..439c5a70f4 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexPlannerTest.java @@ -56,6 +56,7 @@ import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexPlanner.PropertyIndexResult; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReader; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader; import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; @@ -951,6 +952,130 @@ public void relativeProperty_MultipleMatch() throws Exception{ assertFalse(pr.hasProperty("metadata/baz")); } + //~------------------------------< sync indexes > + + @Test + public void syncIndex_uniqueIndex() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn, 100); + + FilterImpl filter = createFilter("nt:base"); + filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.emptyList()); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + assertEquals(1, plan.getEstimatedEntryCount()); + PropertyIndexResult hr = pr(plan).getPropertyIndexResult(); + + assertNotNull(hr); + assertEquals("foo", hr.propertyName); + assertEquals("foo", hr.pr.propertyName); + } + + @Test + public void syncIndex_uniqueAndRelative() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn); + + FilterImpl filter = createFilter("nt:base"); + filter.restrictProperty("jcr:content/foo", Operator.EQUAL, PropertyValues.newString("bar")); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.emptyList()); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + assertEquals(1, plan.getEstimatedEntryCount()); + PropertyIndexResult hr = pr(plan).getPropertyIndexResult(); + + assertNotNull(hr); + assertEquals("foo", hr.propertyName); + assertEquals("jcr:content/foo", hr.pr.propertyName); + } + + @Test + public void syncIndex_nonUnique() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn, 100); + + FilterImpl filter = createFilter("nt:base"); + filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.emptyList()); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + //For non unique count is actual + assertEquals(100, plan.getEstimatedEntryCount()); + PropertyIndexResult hr = pr(plan).getPropertyIndexResult(); + + assertNotNull(hr); + assertEquals("foo", hr.propertyName); + assertEquals("foo", hr.pr.propertyName); + } + + /** + * If both non unique and unique indexes are found then unique should be picked + */ + @Test + public void syncIndex_nonUniqueAndUniqueBoth() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + defnb.indexRule("nt:base").property("bar").propertyIndex().sync(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn, 100); + + FilterImpl filter = createFilter("nt:base"); + filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); + filter.restrictProperty("bar", Operator.EQUAL, PropertyValues.newString("foo")); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, Collections.emptyList()); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + assertEquals(1, plan.getEstimatedEntryCount()); + PropertyIndexResult hr = pr(plan).getPropertyIndexResult(); + + assertNotNull(hr); + assertEquals("foo", hr.propertyName); + assertEquals("foo", hr.pr.propertyName); + } + + @Test + public void syncIndex_NotUsedWithSort() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + defnb.indexRule("nt:base").property("bar").propertyIndex().ordered(); + + IndexDefinition defn = new IndexDefinition(root, defnb.build(), "/foo"); + IndexNode node = createIndexNode(defn, 100); + + FilterImpl filter = createFilter("nt:base"); + filter.restrictProperty("foo", Operator.EQUAL, PropertyValues.newString("bar")); + + IndexPlanner planner = new IndexPlanner(node, "/foo", filter, + ImmutableList.of(new OrderEntry("bar", Type.LONG, OrderEntry.Order.ASCENDING))); + QueryIndex.IndexPlan plan = planner.getPlan(); + assertNotNull(plan); + + assertEquals(100, plan.getEstimatedEntryCount()); + PropertyIndexResult hr = pr(plan).getPropertyIndexResult(); + + assertNull(hr); + } + + private IndexPlanner createPlannerForFulltext(NodeState defn, FullTextExpression exp) throws IOException { IndexNode node = createIndexNode(new IndexDefinition(root, defn, "/foo")); FilterImpl filter = createFilter("nt:base"); diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java index c818afa583..b1f6fe3e8a 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTrackerTest.java @@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; import org.apache.jackrabbit.oak.plugins.index.TrackingCorruptIndexHandler; import org.apache.jackrabbit.oak.plugins.memory.ArrayBasedBlob; @@ -47,6 +48,9 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; @SuppressWarnings("UnusedAssignment") public class IndexTrackerTest { @@ -203,6 +207,50 @@ public void notifyFailedIndexing() throws Exception{ assertTrue(corruptIndexHandler.getFailingIndexData("async").containsKey("/oak:index/foo")); } + + @Test + public void avoidRedundantDiff() throws Exception{ + IndexTracker tracker2 = new IndexTracker(); + + NodeBuilder index = builder.child(INDEX_DEFINITIONS_NAME); + newLucenePropertyIndexDefinition(index, "lucene", ImmutableSet.of("foo"), "async"); + + NodeState before = builder.getNodeState(); + builder.setProperty("foo", "bar"); + NodeState after = builder.getNodeState(); + + NodeState indexed = hook.processCommit(before, after, CommitInfo.EMPTY); + + tracker.update(indexed); + tracker2.update(indexed); + + IndexNode indexNode = tracker.acquireIndexNode("/oak:index/lucene"); + assertEquals(1, indexNode.getSearcher().getIndexReader().numDocs()); + indexNode.release(); + + before = indexed; + builder = before.builder(); + builder.child("a").setProperty("foo", "bar"); + after = builder.getNodeState(); + + AsyncIndexInfoService service = mock(AsyncIndexInfoService.class); + when(service.hasIndexerUpdatedForAnyLane(any(NodeState.class), any(NodeState.class))).thenReturn(false); + tracker.setAsyncIndexInfoService(service); + + indexed = hook.processCommit(before, after, CommitInfo.EMPTY); + tracker.update(indexed); + tracker2.update(indexed); + + //As we falsely said no change has happened index state would remain same + indexNode = tracker.acquireIndexNode("/oak:index/lucene"); + assertEquals(1, indexNode.getSearcher().getIndexReader().numDocs()); + indexNode.release(); + + //While tracker2 does not use async service it sees the index change + indexNode = tracker2.acquireIndexNode("/oak:index/lucene"); + assertEquals(2, indexNode.getSearcher().getIndexReader().numDocs()); + indexNode.release(); + } private NodeState corruptIndex(String indexPath) { NodeBuilder dir = TestUtil.child(builder, PathUtils.concat(indexPath, ":data")); diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java new file mode 100644 index 0000000000..7cde55c265 --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditor2Test.java @@ -0,0 +1,346 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene; + +import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.IndexCommitCallback; +import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.IndexingContext; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter; +import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterFactory; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.lucene.index.IndexableField; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class LuceneIndexEditor2Test { + + private NodeState root = INITIAL_CONTENT; + private NodeState before = root; + private IndexUpdateCallback updateCallback = mock(IndexUpdateCallback.class); + private ExtractedTextCache extractedTextCache = new ExtractedTextCache(0, 0); + private TestIndexingContext indexingContext = new TestIndexingContext(); + private TestWriterFactory writerFactory = new TestWriterFactory(); + private TestPropertyUpdateCallback propCallback = new TestPropertyUpdateCallback(); + private TestWriter writer = new TestWriter(); + private String indexPath = "/oak:index/fooIndex"; + + @Test + public void basics() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + NodeBuilder builder = before.builder(); + builder.child("a").setProperty("foo", "bar"); + + hook.processCommit(root, builder.getNodeState(), CommitInfo.EMPTY); + + assertThat(writer.docs.keySet(), containsInAnyOrder("/a")); + } + + @Test + public void simplePropertyUpdateCallback() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex(); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + ctx.setPropertyUpdateCallback(propCallback); + + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + //Property added + NodeBuilder builder = before.builder(); + builder.child("a").setProperty("foo", "bar"); + builder.child("a").setProperty("foo2", "bar"); + builder.child("a").child("b"); + + before = hook.processCommit(root, builder.getNodeState(), CommitInfo.EMPTY); + propCallback.state.assertState("/a", "foo", UpdateState.ADDED); + assertEquals(1, propCallback.invocationCount); + assertEquals(1, propCallback.doneInvocationCount); + propCallback.reset(); + + //Property updated + builder = before.builder(); + builder.child("a").setProperty("foo", "bar2"); + builder.child("a").setProperty("foo2", "bar2"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + + propCallback.state.assertState("/a", "foo", UpdateState.UPDATED); + + assertEquals(1, propCallback.invocationCount); + propCallback.reset(); + + //Property deleted + builder = before.builder(); + builder.child("a").removeProperty("foo"); + builder.child("a").removeProperty("foo2"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + + propCallback.state.assertState("/a", "foo", UpdateState.DELETED); + assertEquals(1, propCallback.invocationCount); + propCallback.reset(); + } + + @Test + public void relativeProperties() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("jcr:content/metadata/foo").propertyIndex(); + defnb.aggregateRule("nt:base").include("*"); + + NodeState defnState = defnb.build(); + IndexDefinition defn = new IndexDefinition(root, defnState, indexPath); + LuceneIndexEditorContext ctx = newContext(defnState.builder(), defn, true); + ctx.setPropertyUpdateCallback(propCallback); + + EditorHook hook = createHook(ctx); + + updateBefore(defnb); + + //Property added + NodeBuilder builder = before.builder(); + builder.child("a").child("jcr:content").child("metadata").setProperty("foo", "bar"); + builder.child("a").setProperty("foo2", "bar"); + + before = hook.processCommit(root, builder.getNodeState(), CommitInfo.EMPTY); + propCallback.state.assertState("/a", "jcr:content/metadata/foo", UpdateState.ADDED); + assertEquals(1, propCallback.invocationCount); + propCallback.reset(); + + //Property updated + builder = before.builder(); + builder.child("a").child("jcr:content").child("metadata").setProperty("foo", "bar2"); + builder.child("a").setProperty("foo2", "bar2"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + + propCallback.state.assertState("/a", "jcr:content/metadata/foo", UpdateState.UPDATED); + + assertEquals(1, propCallback.invocationCount); + propCallback.reset(); + + //Property deleted + builder = before.builder(); + builder.child("a").child("jcr:content").child("metadata").removeProperty("foo"); + builder.child("a").removeProperty("foo2"); + before = hook.processCommit(before, builder.getNodeState(), CommitInfo.EMPTY); + + propCallback.state.assertState("/a", "jcr:content/metadata/foo", UpdateState.DELETED); + assertEquals(1, propCallback.invocationCount); + propCallback.reset(); + } + + private void updateBefore(IndexDefinitionBuilder defnb) { + NodeBuilder builder = before.builder(); + NodeBuilder cb = TestUtil.child(builder, PathUtils.getParentPath(indexPath)); + cb.setChildNode(PathUtils.getName(indexPath), defnb.build()); + before = builder.getNodeState(); + } + + private EditorHook createHook(LuceneIndexEditorContext context) { + IndexEditorProvider provider = new IndexEditorProvider() { + @CheckForNull + @Override + public Editor getIndexEditor(@Nonnull String type, @Nonnull NodeBuilder definition, + @Nonnull NodeState root, @Nonnull IndexUpdateCallback callback) + throws CommitFailedException { + if ("lucene".equals(type)) { + return new LuceneIndexEditor(context); + } + return null; + } + }; + + String async = context.isAsyncIndexing() ? "async" : null; + IndexUpdateProvider updateProvider = new IndexUpdateProvider(provider, async, false); + return new EditorHook(updateProvider); + } + + private LuceneIndexEditorContext newContext(NodeBuilder defnBuilder, IndexDefinition defn, boolean asyncIndex) { + return new LuceneIndexEditorContext(root, defnBuilder, defn, updateCallback, writerFactory, + extractedTextCache, null, indexingContext, asyncIndex); + } + + + private static class TestPropertyUpdateCallback implements PropertyUpdateCallback { + int invocationCount; + CallbackState state; + int doneInvocationCount; + + @Override + public void propertyUpdated(String nodePath, String propertyRelativePath, PropertyDefinition pd, + PropertyState before, PropertyState after) { + assertNotNull(nodePath); + assertNotNull(propertyRelativePath); + assertNotNull(pd); + + if (before == null && after == null) { + fail("Both states cannot be null at same time"); + } + + state = new CallbackState(nodePath, propertyRelativePath, pd, before, after); + invocationCount++; + } + + @Override + public void done() throws CommitFailedException { + doneInvocationCount++; + } + + void reset(){ + state = null; + invocationCount = 0; + doneInvocationCount = 0; + } + } + + enum UpdateState {ADDED, UPDATED, DELETED} + + private static class CallbackState { + final String nodePath; + final String propertyPath; + final PropertyDefinition pd; + final PropertyState before; + final PropertyState after; + + + public CallbackState(String nodePath, String propertyPath, PropertyDefinition pd, + PropertyState before, PropertyState after) { + this.nodePath = nodePath; + this.propertyPath = propertyPath; + this.pd = pd; + this.before = before; + this.after = after; + } + + public void assertState(String expectedPath, String expectedName, UpdateState us) { + assertEquals(expectedPath, nodePath); + assertEquals(expectedName, propertyPath); + + switch (us) { + case ADDED: assertNotNull(after); assertNull(before); break; + case UPDATED: assertNotNull(after); assertNotNull(before); break; + case DELETED: assertNull(after); assertNotNull(before); break; + } + } + } + + + private class TestWriterFactory implements LuceneIndexWriterFactory { + @Override + public LuceneIndexWriter newInstance(IndexDefinition definition, + NodeBuilder definitionBuilder, boolean reindex) { + return writer; + } + } + + private static class TestWriter implements LuceneIndexWriter { + Set deletedPaths = new HashSet<>(); + Map> docs = new HashMap<>(); + boolean closed; + + @Override + public void updateDocument(String path, Iterable doc) throws IOException { + docs.put(path, doc); + } + + @Override + public void deleteDocuments(String path) throws IOException { + deletedPaths.add(path); + } + + @Override + public boolean close(long timestamp) throws IOException { + closed = true; + return true; + } + } + + private class TestIndexingContext implements IndexingContext { + CommitInfo info = CommitInfo.EMPTY; + boolean reindexing; + boolean async; + + @Override + public String getIndexPath() { + return indexPath; + } + + @Override + public CommitInfo getCommitInfo() { + return info; + } + + @Override + public boolean isReindexing() { + return reindexing; + } + + @Override + public boolean isAsync() { + return async; + } + + @Override + public void indexUpdateFailed(Exception e) { + + } + + @Override + public void registerIndexCommitCallback(IndexCommitCallback callback) { + + } + } +} diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java index f4b1da0379..6b55b37a1a 100644 --- a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexProviderServiceTest.java @@ -38,6 +38,7 @@ import org.apache.commons.lang3.reflect.FieldUtils; import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean; import org.apache.jackrabbit.oak.api.jmx.CheckpointMBean; +import org.apache.jackrabbit.oak.osgi.OsgiWhiteboard; import org.apache.jackrabbit.oak.plugins.blob.datastore.CachingFileDataStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore; import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreUtils; @@ -47,6 +48,7 @@ import org.apache.jackrabbit.oak.plugins.index.IndexPathService; import org.apache.jackrabbit.oak.plugins.index.fulltext.PreExtractedTextProvider; import org.apache.jackrabbit.oak.plugins.index.importer.IndexImporterProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner; import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory; import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore; @@ -56,6 +58,8 @@ import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.InfoStream; @@ -84,6 +88,8 @@ private LuceneIndexProviderService service = new LuceneIndexProviderService(); + private Whiteboard wb; + @Before public void setUp(){ context.registerService(MountInfoProvider.class, Mounts.defaultMountInfoProvider()); @@ -94,6 +100,8 @@ public void setUp(){ context.registerService(IndexPathService.class, mock(IndexPathService.class)); context.registerService(AsyncIndexInfoService.class, mock(AsyncIndexInfoService.class)); context.registerService(CheckpointMBean.class, mock(CheckpointMBean.class)); + + wb = new OsgiWhiteboard(context.bundleContext()); MockOsgi.injectServices(service, context.bundleContext()); } @@ -133,7 +141,12 @@ public void defaultSetup() throws Exception{ assertNotNull(context.getService(JournalPropertyService.class)); assertNotNull(context.getService(IndexImporterProvider.class)); + assertNotNull(WhiteboardUtils.getServices(wb, Runnable.class, r -> r instanceof PropertyIndexCleaner)); + MockOsgi.deactivate(service, context.bundleContext()); + + IndexTracker tracker = (IndexTracker) FieldUtils.readDeclaredField(service, "tracker", true); + assertNotNull(tracker.getAsyncIndexInfoService()); } @Test @@ -316,6 +329,29 @@ public Object call() throws Exception { } + @Test + public void cleanerRegistration() throws Exception{ + Map config = getDefaultConfig(); + config.put("propIndexCleanerIntervalInSecs", 142); + + MockOsgi.activate(service, context.bundleContext(), config); + ServiceReference[] sr = context.bundleContext().getAllServiceReferences(Runnable.class.getName(), + "(scheduler.name="+PropertyIndexCleaner.class.getName()+")"); + assertEquals(sr.length, 1); + + assertEquals(142L, sr[0].getProperty("scheduler.period")); + } + + @Test + public void cleanerRegistrationDisabled() throws Exception{ + Map config = getDefaultConfig(); + config.put("propIndexCleanerIntervalInSecs", 0); + + MockOsgi.activate(service, context.bundleContext(), config); + ServiceReference[] sr = context.bundleContext().getAllServiceReferences(Runnable.class.getName(), + "(scheduler.name="+PropertyIndexCleaner.class.getName()+")"); + assertNull(sr); + } private void reactivate() { MockOsgi.deactivate(service, context.bundleContext()); diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcherTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcherTest.java new file mode 100644 index 0000000000..f62513989e --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/BucketSwitcherTest.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import org.apache.jackrabbit.oak.spi.state.EqualsDiff; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static com.google.common.collect.ImmutableList.copyOf; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_HEAD_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_PREVIOUS_BUCKET; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +public class BucketSwitcherTest { + private NodeBuilder builder = EMPTY_NODE.builder(); + private BucketSwitcher bs = new BucketSwitcher(builder); + + @Test + public void basic() throws Exception { + bs.switchBucket(100); + assertThat(copyOf(bs.getOldBuckets()), empty()); + } + + @Test + public void singleUnusedBucket() throws Exception { + builder.child("1"); + builder.setProperty(PROP_HEAD_BUCKET, "1"); + + bs.switchBucket(100); + assertThat(copyOf(bs.getOldBuckets()), empty()); + } + + @Test + public void twoBucket_HeadUnused() throws Exception { + builder.child("1"); + builder.child("2"); + builder.setProperty(PROP_HEAD_BUCKET, "2"); + builder.setProperty(PROP_PREVIOUS_BUCKET, "1"); + + bs.switchBucket(100); + assertFalse(builder.hasProperty(PROP_PREVIOUS_BUCKET)); + assertEquals("2", builder.getString(PROP_HEAD_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + } + + @Test + public void twoBuckets_BothUsed() throws Exception { + builder.child("2").child("foo"); + builder.child("1"); + builder.setProperty(PROP_HEAD_BUCKET, "2"); + builder.setProperty(PROP_PREVIOUS_BUCKET, "1"); + + bs.switchBucket(100); + + assertEquals("2", builder.getString(PROP_PREVIOUS_BUCKET)); + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertTrue(builder.hasChildNode("3")); + + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + } + + @Test + public void twoBuckets_2Switches() throws Exception{ + builder.child("2").child("foo"); + builder.child("1"); + builder.setProperty(PROP_HEAD_BUCKET, "2"); + builder.setProperty(PROP_PREVIOUS_BUCKET, "1"); + + bs.switchBucket(100); + + bs.switchBucket(150); + assertFalse(builder.hasProperty(PROP_PREVIOUS_BUCKET)); + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1", "2")); + } + + /** + * Test the case where lastIndexedTo time does not change i.e. async indexer + * has not moved on between different run. In such a case the property index + * state should remain same and both head and previous bucket should not be + * changed + */ + @Test + public void twoBuckets_NoChange() throws Exception{ + builder.child("2").child("foo"); + builder.child("1"); + builder.setProperty(PROP_HEAD_BUCKET, "2"); + builder.setProperty(PROP_PREVIOUS_BUCKET, "1"); + + NodeState state0 = builder.getNodeState(); + assertTrue(bs.switchBucket(100)); + + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertEquals("2", builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + + NodeState state1 = builder.getNodeState(); + assertFalse(bs.switchBucket(100)); + + //No changed in async indexer state so current bucket state would remain + //as previous + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertEquals("2", builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + + NodeState state2 = builder.getNodeState(); + assertFalse(bs.switchBucket(100)); + + //Async indexer time still not changed. So head bucket remains same + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertEquals("2", builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + + NodeState state3 = builder.getNodeState(); + + assertTrue(EqualsDiff.equals(state1, state2)); + assertTrue(EqualsDiff.equals(state2, state3)); + } + + /** + * Test the case where async indexer state changes i.e. lastIndexedTo changes + * however nothing new got indexed in property index. In such a case + * after the second run there should be no previous bucket + */ + @Test + public void twoBucket_IndexedToTimeChange() throws Exception{ + builder.child("2").child("foo"); + builder.child("1"); + builder.setProperty(PROP_HEAD_BUCKET, "2"); + builder.setProperty(PROP_PREVIOUS_BUCKET, "1"); + + NodeState state0 = builder.getNodeState(); + assertTrue(bs.switchBucket(100)); + + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertEquals("2", builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1")); + + NodeState state1 = builder.getNodeState(); + assertTrue(bs.switchBucket(150)); + + //This time previous bucket should be discarded + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertNull(builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1", "2")); + + NodeState state2 = builder.getNodeState(); + assertTrue(bs.switchBucket(200)); + + assertEquals("3", builder.getString(PROP_HEAD_BUCKET)); + assertNull(builder.getString(PROP_PREVIOUS_BUCKET)); + assertThat(copyOf(bs.getOldBuckets()), containsInAnyOrder("1", "2")); + + //assert no change done after previous is removed + //not even change of asyncIndexedTo + NodeState state3 = builder.getNodeState(); + assertTrue(EqualsDiff.equals(state2, state3)); + } + +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookupTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookupTest.java new file mode 100644 index 0000000000..72b2c7bc9e --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexLookupTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.Cursors; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.query.NodeStateNodeTypeInfoProvider; +import org.apache.jackrabbit.oak.query.QueryEngineSettings; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider; +import org.apache.jackrabbit.oak.query.ast.Operator; +import org.apache.jackrabbit.oak.query.ast.SelectorImpl; +import org.apache.jackrabbit.oak.query.index.FilterImpl; +import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.query.Filter; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyValues.newString; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertThat; + +public class HybridPropertyIndexLookupTest { + private NodeState root = INITIAL_CONTENT; + private NodeBuilder builder = EMPTY_NODE.builder(); + private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + private String indexPath = "/oak:index/foo"; + private PropertyIndexUpdateCallback callback = new PropertyIndexUpdateCallback(indexPath, builder); + + @Test + public void simplePropertyRestriction() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync(); + + propertyUpdated("/a", "foo", "bar"); + + FilterImpl f = createFilter(); + f.restrictProperty("foo", Operator.EQUAL, newString("bar")); + + assertThat(query(f, "foo"), containsInAnyOrder("/a")); + } + + @Test + public void valuePattern() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync().valuePattern("(a.*|b)"); + + propertyUpdated("/a", "foo", "a"); + propertyUpdated("/a1", "foo", "a1"); + propertyUpdated("/b", "foo", "b"); + propertyUpdated("/c", "foo", "c"); + + assertThat(query("foo", "a"), containsInAnyOrder("/a")); + assertThat(query("foo", "a1"), containsInAnyOrder("/a1")); + assertThat(query("foo", "b"), containsInAnyOrder("/b")); + + // c should not be found as its excluded + assertThat(query("foo", "c"), empty()); + } + + @Test + public void relativeProperty() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync(); + + propertyUpdated("/a", "foo", "bar"); + + FilterImpl f = createFilter(); + f.restrictProperty("jcr:content/foo", Operator.EQUAL, newString("bar")); + + assertThat(query(f, "foo", "jcr:content/foo"), containsInAnyOrder("/a")); + } + + @Test + public void pathResultAbsolutePath() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync(); + + propertyUpdated("/a", "foo", "bar"); + + String propertyName = "foo"; + FilterImpl filter = createFilter(); + filter.restrictProperty("foo", Operator.EQUAL, newString("bar")); + + HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(indexPath, builder.getNodeState()); + Iterable paths = lookup.query(filter, pd(propertyName), propertyName, + filter.getPropertyRestriction(propertyName)); + + assertThat(ImmutableList.copyOf(paths), containsInAnyOrder("/a")); + } + + private void propertyUpdated(String nodePath, String propertyRelativeName, String value){ + callback.propertyUpdated(nodePath, propertyRelativeName, pd(propertyRelativeName), + null, createProperty(PathUtils.getName(propertyRelativeName), value)); + } + + private List query(String propertyName, String value) { + FilterImpl f = createFilter(); + f.restrictProperty(propertyName, Operator.EQUAL, newString(value)); + return query(f, propertyName); + } + + private List query(Filter filter, String propertyName) { + return query(filter, propertyName, propertyName); + } + + private List query(Filter filter, String propertyName, String propertyRestrictionName) { + HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(indexPath, builder.getNodeState()); + Iterable paths = lookup.query(filter, pd(propertyName), propertyName, + filter.getPropertyRestriction(propertyRestrictionName)); + return ImmutableList.copyOf(paths); + } + + private PropertyDefinition pd(String propName){ + IndexDefinition defn = new IndexDefinition(root, defnb.build(), indexPath); + return defn.getApplicableIndexingRule("nt:base").getConfig(propName); + } + + private FilterImpl createFilter() { + return createFilter(root, "nt:base"); + } + + private FilterImpl createFilter(NodeState root, String nodeTypeName) { + NodeTypeInfoProvider nodeTypes = new NodeStateNodeTypeInfoProvider(root); + NodeTypeInfo type = nodeTypes.getNodeTypeInfo(nodeTypeName); + SelectorImpl selector = new SelectorImpl(type, nodeTypeName); + return new FilterImpl(selector, "SELECT * FROM [" + nodeTypeName + "]", new QueryEngineSettings()); + } + +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexStorageTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexStorageTest.java new file mode 100644 index 0000000000..0f21441c53 --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/HybridPropertyIndexStorageTest.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; +import org.apache.jackrabbit.oak.api.PropertyValue; +import org.apache.jackrabbit.oak.plugins.index.Cursors; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.query.NodeStateNodeTypeInfoProvider; +import org.apache.jackrabbit.oak.query.QueryEngineSettings; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfo; +import org.apache.jackrabbit.oak.query.ast.NodeTypeInfoProvider; +import org.apache.jackrabbit.oak.query.ast.SelectorImpl; +import org.apache.jackrabbit.oak.query.index.FilterImpl; +import org.apache.jackrabbit.oak.spi.query.Cursor; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROPERTY_INDEX; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_HEAD_BUCKET; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_PREVIOUS_BUCKET; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyValues.newString; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +public class HybridPropertyIndexStorageTest { + private NodeState root = INITIAL_CONTENT; + private NodeBuilder builder = EMPTY_NODE.builder(); + private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + private String indexPath = "/oak:index/foo"; + + @Test + public void nonSyncProp() throws Exception{ + defnb.indexRule("nt:base").property("foo"); + + newCallback().propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + + assertFalse(builder.isModified()); + } + + @Test + public void simpleProperty() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync(); + + newCallback().propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + newCallback().propertyUpdated("/b", "foo", pd("foo"), + null, createProperty("foo", "bar2")); + + assertThat(query("foo", newString("bar")), containsInAnyOrder("/a")); + assertThat(query("foo", newString("bar2")), containsInAnyOrder("/b")); + } + + @Test + public void relativeProperty() throws Exception{ + String propName = "jcr:content/foo"; + defnb.indexRule("nt:base").property(propName).sync(); + + newCallback().propertyUpdated("/a", propName, pd(propName), + null, createProperty("foo", "bar")); + + assertThat(query(propName, newString("bar")), containsInAnyOrder("/a")); + } + + @Test + public void valuePattern() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync().valueIncludedPrefixes("bar/"); + + newCallback().propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar/a")); + newCallback().propertyUpdated("/b", "foo", pd("foo"), + null, createProperty("foo", "baz/a")); + + assertThat(query("foo", newString("bar/a")), containsInAnyOrder("/a")); + + //As baz pattern is excluded it should not be indexed + assertThat(query("foo", newString("baz/a")), empty()); + } + + @Test + public void pruningDisabledForSimpleProperty() throws Exception{ + defnb.indexRule("nt:base").property("foo").sync(); + + newCallback().propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + newCallback().propertyUpdated("/b", "foo", pd("foo"), + null, createProperty("foo", "bar")); + + assertThat(query("foo", newString("bar")), containsInAnyOrder("/a", "/b")); + + builder = builder.getNodeState().builder(); + newCallback().propertyUpdated("/b", "foo", pd("foo"), + createProperty("foo", "bar"), null); + + // /b would still come as pruning is disabled + assertThat(query("foo", newString("bar")), containsInAnyOrder("/a", "/b")); + } + + //~----------------------------------------< unique props > + + @Test + public void uniqueProperty() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + PropertyUpdateCallback callback = newCallback(); + + callback.propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + callback.propertyUpdated("/b", "foo", pd("foo"), + null, createProperty("foo", "bar2")); + + callback.done(); + + assertThat(query("foo", newString("bar")), containsInAnyOrder("/a")); + } + + @Test + public void pruningWorkingForUnique() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + newCallback().propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + + assertThat(query("foo", newString("bar")), containsInAnyOrder("/a")); + + builder = builder.getNodeState().builder(); + newCallback().propertyUpdated("/a", "foo", pd("foo"), + createProperty("foo", "bar"), null); + + // /b should not come as pruning is enabled + assertThat(query("foo", newString("bar")), empty()); + } + + //~---------------------------------------< buckets > + + @Test + public void bucketSwitch() throws Exception{ + String propName = "foo"; + defnb.indexRule("nt:base").property(propName).sync(); + + newCallback().propertyUpdated("/a", propName, pd(propName), + null, createProperty(propName, "bar")); + + assertThat(query(propName, newString("bar")), containsInAnyOrder("/a")); + + switchBucket(propName); + + newCallback().propertyUpdated("/b", propName, pd(propName), + null, createProperty(propName, "bar")); + + assertThat(query(propName, newString("bar")), containsInAnyOrder("/a", "/b")); + + switchBucket(propName); + + newCallback().propertyUpdated("/c", propName, pd(propName), + null, createProperty(propName, "bar")); + + //Now /a should not come as its in 3rd bucket and we only consider head and previous buckets + assertThat(query(propName, newString("bar")), containsInAnyOrder("/b", "/c")); + } + + private void switchBucket(String propertyName) { + NodeBuilder propertyIndex = builder.child(PROPERTY_INDEX); + NodeBuilder idx = propertyIndex.child(HybridPropertyIndexUtil.getNodeName(propertyName)); + + String head = idx.getString(HybridPropertyIndexUtil.PROP_HEAD_BUCKET); + assertNotNull(head); + + int id = Integer.parseInt(head); + idx.setProperty(PROP_PREVIOUS_BUCKET, head); + idx.setProperty(PROP_HEAD_BUCKET, String.valueOf(id + 1)); + + builder = builder.getNodeState().builder(); + } + + private List query(String propertyName, PropertyValue value) { + HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(indexPath, builder.getNodeState()); + FilterImpl filter = createFilter(root, "nt:base"); + Iterable paths = lookup.query(filter, pd(propertyName), propertyName, value); + return ImmutableList.copyOf(paths); + } + + private PropertyIndexUpdateCallback newCallback(){ + return new PropertyIndexUpdateCallback(indexPath, builder); + } + + private PropertyDefinition pd(String propName){ + IndexDefinition defn = new IndexDefinition(root, defnb.build(), indexPath); + return defn.getApplicableIndexingRule("nt:base").getConfig(propName); + } + + private static FilterImpl createFilter(NodeState root, String nodeTypeName) { + NodeTypeInfoProvider nodeTypes = new NodeStateNodeTypeInfoProvider(root); + NodeTypeInfo type = nodeTypes.getNodeTypeInfo(nodeTypeName); + SelectorImpl selector = new SelectorImpl(type, nodeTypeName); + return new FilterImpl(selector, "SELECT * FROM [" + nodeTypeName + "]", new QueryEngineSettings()); + } + +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQueryTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQueryTest.java new file mode 100644 index 0000000000..1dddede256 --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/LuceneIndexPropertyQueryTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import com.google.common.collect.Iterables; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EditorHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.child; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.*; + +public class LuceneIndexPropertyQueryTest { + private NodeState root = INITIAL_CONTENT; + private NodeBuilder builder = root.builder(); + private IndexTracker tracker = new IndexTracker(); + + private String indexPath = "/oak:index/foo"; + private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(child(builder, indexPath)); + + private LuceneIndexPropertyQuery query = new LuceneIndexPropertyQuery(tracker, indexPath); + + @Test + public void simplePropertyIndex() throws Exception{ + defnb.noAsync(); + defnb.indexRule("nt:base").property("foo").propertyIndex(); + + assertEquals(0,Iterables.size(query.getIndexedPaths("foo", "bar"))); + + NodeState before = builder.getNodeState(); + builder.child("a").setProperty("foo", "bar"); + builder.child("b").setProperty("foo", "bar"); + NodeState after = builder.getNodeState(); + + EditorHook hook = new EditorHook(new IndexUpdateProvider(new LuceneIndexEditorProvider())); + NodeState indexed = hook.processCommit(before, after, CommitInfo.EMPTY); + tracker.update(indexed); + + assertThat(query.getIndexedPaths("foo", "bar"), + containsInAnyOrder("/a", "/b")); + } +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java new file mode 100644 index 0000000000..2f25639607 --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/PropertyIndexCleanerTest.java @@ -0,0 +1,308 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.annotation.CheckForNull; + +import com.google.common.collect.ImmutableList; +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfo; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.plugins.memory.PropertyValues; +import org.apache.jackrabbit.oak.query.index.FilterImpl; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.stats.Clock; +import org.junit.Before; +import org.junit.Test; + +import static java.util.Arrays.asList; +import static org.apache.jackrabbit.oak.api.CommitFailedException.CONSTRAINT; +import static org.apache.jackrabbit.oak.commons.PathUtils.getName; +import static org.apache.jackrabbit.oak.commons.PathUtils.getParentPath; +import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.child; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.apache.jackrabbit.oak.spi.state.NodeStateUtils.getNode; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class PropertyIndexCleanerTest { + private NodeStore nodeStore = new MemoryNodeStore(); + private SimpleAsyncInfoService asyncService = new SimpleAsyncInfoService(); + private Clock clock = new Clock.Virtual(); + + @Before + public void setUp() throws CommitFailedException { + NodeBuilder nb = nodeStore.getRoot().builder(); + new InitialContent().initialize(nb); + merge(nb); + } + + @Test + public void syncIndexPaths() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + String indexPath = "/oak:index/foo"; + addIndex(indexPath, defnb); + + PropertyIndexCleaner cleaner = + new PropertyIndexCleaner(nodeStore, () -> asList("/oak:index/uuid", indexPath), asyncService); + + //As index is yet not update it would not show up in sync index paths + assertThat(cleaner.getSyncIndexPaths(), empty()); + + NodeBuilder builder = nodeStore.getRoot().builder(); + PropertyIndexUpdateCallback cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/a", "foo", "bar"); + merge(builder); + + //Post update it would show up + assertThat(cleaner.getSyncIndexPaths(), containsInAnyOrder(indexPath)); + } + + @Test + public void simplePropertyIndexCleaning() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + String indexPath = "/oak:index/foo"; + addIndex(indexPath, defnb); + + PropertyIndexCleaner cleaner = + new PropertyIndexCleaner(nodeStore, () -> asList("/oak:index/uuid", indexPath), asyncService); + + NodeBuilder builder = nodeStore.getRoot().builder(); + PropertyIndexUpdateCallback cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/a", "foo", "bar"); + merge(builder); + + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a")); + + //------------------------ Run 1 + asyncService.addInfo("async", 1000); + assertTrue(cleaner.performCleanup()); + + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a")); + + builder = nodeStore.getRoot().builder(); + cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/b", "foo", "bar"); + merge(builder); + + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a", "/b")); + + //------------------------ Run 2 + asyncService.addInfo("async", 2000); + assertTrue(cleaner.performCleanup()); + + //Now /a would be part of removed bucket + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/b")); + + //------------------------ Run 3 + asyncService.addInfo("async", 3000); + assertTrue(cleaner.performCleanup()); + + //With another run /b would also be removed + assertThat(query(indexPath, "foo", "bar"), empty()); + } + + @Test + public void uniqueIndexCleaning() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + String indexPath = "/oak:index/foo"; + addIndex(indexPath, defnb); + + PropertyIndexCleaner cleaner = + new PropertyIndexCleaner(nodeStore, () -> asList("/oak:index/uuid", indexPath), asyncService); + cleaner.setCreatedTimeThreshold(TimeUnit.MILLISECONDS, 100); + + clock.waitUntil(1000); + + NodeBuilder builder = nodeStore.getRoot().builder(); + PropertyIndexUpdateCallback cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/a", "foo", "bar"); + cb.done(); + merge(builder); + + clock.waitUntil(1150); + + builder = nodeStore.getRoot().builder(); + cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/b", "foo", "bar2"); + cb.done(); + merge(builder); + + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a")); + assertThat(query(indexPath, "foo", "bar2"), containsInAnyOrder("/b")); + + //------------------------ Run 1 + asyncService.addInfo("async", 1200); + assertTrue(cleaner.performCleanup()); + + // /a would be purged, /b would be retained as its created time 1150 is not older than 100 wrt + // indexer time of 1200 + assertThat(query(indexPath, "foo", "bar"), empty()); + assertThat(query(indexPath, "foo", "bar2"), containsInAnyOrder("/b")); + + builder = nodeStore.getRoot().builder(); + cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/c", "foo", "bar2"); + + try{ + cb.done(); + fail(); + } catch (CommitFailedException e){ + assertEquals(CONSTRAINT,e.getType()); + } + + //------------------------ Run 2 + asyncService.addInfo("async", 1400); + assertTrue(cleaner.performCleanup()); + + //Both entries would have been purged + assertThat(query(indexPath, "foo", "bar"), empty()); + assertThat(query(indexPath, "foo", "bar2"), empty()); + } + + @Test + public void noRunPerformedIfNoChangeInAsync() throws Exception{ + IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + String indexPath = "/oak:index/foo"; + addIndex(indexPath, defnb); + + PropertyIndexCleaner cleaner = + new PropertyIndexCleaner(nodeStore, () -> asList("/oak:index/uuid", indexPath), asyncService); + + NodeBuilder builder = nodeStore.getRoot().builder(); + PropertyIndexUpdateCallback cb = newCallback(builder, indexPath); + propertyUpdated(cb, indexPath, "/a", "foo", "bar"); + merge(builder); + + assertThat(query(indexPath, "foo", "bar"), containsInAnyOrder("/a")); + + //------------------------ Run 1 + asyncService.addInfo("async", 1000); + assertTrue(cleaner.performCleanup()); + + //Second run should not run + assertFalse(cleaner.performCleanup()); + } + + private void addIndex(String indexPath, IndexDefinitionBuilder defnb) throws CommitFailedException { + NodeBuilder nb = nodeStore.getRoot().builder(); + child(nb, getParentPath(indexPath)).setChildNode(getName(indexPath), defnb.build()); + merge(nb); + } + + private void propertyUpdated(PropertyUpdateCallback callback, String indexPath, String nodePath, String propertyName, + String value){ + callback.propertyUpdated(nodePath, propertyName, pd(indexPath, propertyName), + null, createProperty(propertyName, value)); + } + + private PropertyIndexUpdateCallback newCallback(NodeBuilder builder, String indexPath) { + return new PropertyIndexUpdateCallback(indexPath, child(builder, indexPath), clock); + } + + private PropertyDefinition pd(String indexPath, String propName){ + NodeState root = nodeStore.getRoot(); + IndexDefinition defn = new IndexDefinition(root, getNode(root, indexPath), indexPath); + return defn.getApplicableIndexingRule("nt:base").getConfig(propName); + } + + private void merge(NodeBuilder nb) throws CommitFailedException { + nodeStore.merge(nb, EmptyHook.INSTANCE, CommitInfo.EMPTY); + } + + private List query(String indexPath, String propertyName, String value) { + NodeState root = nodeStore.getRoot(); + HybridPropertyIndexLookup lookup = new HybridPropertyIndexLookup(indexPath, getNode(root, indexPath)); + FilterImpl filter = FilterImpl.newTestInstance(); + Iterable paths = lookup.query(filter, pd(indexPath, propertyName), propertyName, + PropertyValues.newString(value)); + return ImmutableList.copyOf(paths); + } + + private static class SimpleAsyncInfoService implements AsyncIndexInfoService { + final Map infos = new HashMap<>(); + + @Override + public Iterable getAsyncLanes() { + return infos.keySet(); + } + + @Override + public Iterable getAsyncLanes(NodeState root) { + throw new UnsupportedOperationException(); + } + + @CheckForNull + @Override + public AsyncIndexInfo getInfo(String name) { + return infos.get(name); + } + + @CheckForNull + @Override + public AsyncIndexInfo getInfo(String name, NodeState root) { + return null; + } + + @Override + public Map getIndexedUptoPerLane() { + Map result = new HashMap<>(); + for (AsyncIndexInfo info : infos.values()) { + result.put(info.getName(), info.getLastIndexedTo()); + } + return result; + } + + @Override + public Map getIndexedUptoPerLane(NodeState root) { + throw new UnsupportedOperationException(); + } + + public void addInfo(String name, long lastIndexedTo) { + infos.put(name, new AsyncIndexInfo(name, lastIndexedTo, 0, false, null)); + } + } + + +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java new file mode 100644 index 0000000000..001f1dbe08 --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java @@ -0,0 +1,281 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import java.io.File; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.IndexingQueue; +import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.query.AbstractQueryTest; +import org.apache.jackrabbit.oak.spi.commit.CommitInfo; +import org.apache.jackrabbit.oak.spi.commit.EmptyHook; +import org.apache.jackrabbit.oak.spi.commit.Observer; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.apache.jackrabbit.oak.stats.Clock; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.apache.jackrabbit.oak.api.CommitFailedException.CONSTRAINT; +import static org.apache.jackrabbit.oak.commons.PathUtils.concat; +import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +public class SynchronousPropertyIndexTest extends AbstractQueryTest { + private ExecutorService executorService = Executors.newFixedThreadPool(2); + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target")); + + private LuceneIndexProvider luceneIndexProvider; + private IndexingQueue queue = mock(IndexingQueue.class); + private NodeStore nodeStore = new MemoryNodeStore(); + private NRTIndexFactory nrtIndexFactory; + private Whiteboard wb; + + + private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + private String indexPath = "/oak:index/foo"; + + @Before + public void setUp(){ + setTraversalEnabled(false); + } + + @After + public void tearDown() throws IOException { + luceneIndexProvider.close(); + new ExecutorCloser(executorService).close(); + } + + @Override + protected ContentRepository createRepository() { + IndexCopier copier; + try { + copier = new IndexCopier(executorService, temporaryFolder.getRoot()); + } catch (IOException e) { + throw new RuntimeException(e); + } + + nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, 1000, StatisticsProvider.NOOP); + MountInfoProvider mip = defaultMountInfoProvider(); + LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier); + IndexTracker tracker = new IndexTracker(indexReaderFactory,nrtIndexFactory); + + luceneIndexProvider = new LuceneIndexProvider(tracker); + LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier, + tracker, + null, + null, + Mounts.defaultMountInfoProvider()); + + editorProvider.setIndexingQueue(queue); + + Oak oak = new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with((QueryIndexProvider) luceneIndexProvider) + .with((Observer) luceneIndexProvider) + .with(editorProvider) + .with(new PropertyIndexEditorProvider()) + .with(new NodeTypeIndexProvider()) + .with(new NodeCounterEditorProvider()) + //Effectively disable async indexing auto run + //such that we can control run timing as per test requirement + .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); + + wb = oak.getWhiteboard(); + return oak.createContentRepository(); + } + + @Test + public void uniquePropertyCommit() throws Exception{ + defnb.async("async", "nrt"); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a").setProperty("foo", "bar"); + root.commit(); + + createPath("/b").setProperty("foo", "bar"); + try { + root.commit(); + fail(); + } catch (CommitFailedException e) { + assertEquals(CONSTRAINT, e.getType()); + } + } + + @Test + public void uniquePropertyCommit_Async() throws Exception{ + defnb.async("async", "nrt"); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a").setProperty("foo", "bar"); + root.commit(); + runAsyncIndex(); + + //Remove the :property-index node to simulate bucket change + //This time commit would trigger a lucene query + NodeBuilder builder = nodeStore.getRoot().builder(); + String propIdxStorePath = concat(indexPath, HybridPropertyIndexUtil.PROPERTY_INDEX); + NodeBuilder propIndex = TestUtil.child(builder, propIdxStorePath); + propIndex.remove(); + nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY); + root.refresh(); + + createPath("/b").setProperty("foo", "bar"); + try { + root.commit(); + fail(); + } catch (CommitFailedException e) { + assertEquals(CONSTRAINT, e.getType()); + } + } + + @Test + public void nonUniqueIndex() throws Exception{ + defnb.async("async", "nrt"); + defnb.indexRule("nt:base").property("foo").propertyIndex().sync(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a").setProperty("foo", "bar"); + root.commit(); + + assertQuery("select * from [nt:base] where [foo] = 'bar'", asList("/a")); + + runAsyncIndex(); + + createPath("/b").setProperty("foo", "bar"); + root.commit(); + + assertQuery("select * from [nt:base] where [foo] = 'bar'", asList("/a", "/b")); + + //Do multiple runs which lead to path being returned from both property and lucene + //index. But the actual result should only contain unique paths + runAsyncIndex(); + runAsyncIndex(); + + assertQuery("select * from [nt:base] where [foo] = 'bar'", asList("/a", "/b")); + } + + @Test + public void uniquePaths() throws Exception{ + defnb.async("async", "nrt"); + defnb.indexRule("nt:base").property("foo").propertyIndex().unique(); + + addIndex(indexPath, defnb); + root.commit(); + + createPath("/a").setProperty("foo", "bar"); + root.commit(); + + assertQuery("select * from [nt:base] where [foo] = 'bar'", singletonList("/a")); + + runAsyncIndex(); + createPath("/b").setProperty("foo", "bar2"); + root.commit(); + + runAsyncIndex(); + createPath("/c").setProperty("foo", "bar3"); + root.commit(); + + assertQuery("select * from [nt:base] where [foo] = 'bar'", singletonList("/a")); + assertQuery("select * from [nt:base] where [foo] = 'bar2'", singletonList("/b")); + assertQuery("select * from [nt:base] where [foo] = 'bar3'", singletonList("/c")); + + createPath("/d").setProperty("foo", "bar"); + try { + root.commit(); + fail(); + } catch (CommitFailedException e) { + assertEquals(CONSTRAINT, e.getType()); + } + } + + private void runAsyncIndex() { + AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, + Runnable.class, input -> input instanceof AsyncIndexUpdate); + assertNotNull(async); + async.run(); + if (async.isFailing()) { + fail("AsyncIndexUpdate failed"); + } + root.refresh(); + } + + private void addIndex(String indexPath, IndexDefinitionBuilder defnb){ + defnb.build(createPath(indexPath)); + } + + private Tree createPath(String path){ + Tree base = root.getTree("/"); + for (String e : PathUtils.elements(path)){ + base = base.hasChild(e) ? base.getChild(e) : base.addChild(e); + } + return base; + } +} diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleanerTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleanerTest.java new file mode 100644 index 0000000000..698810209f --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniqueIndexCleanerTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + + +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.junit.Test; + +import static com.google.common.collect.ImmutableList.copyOf; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexUtil.PROP_CREATED; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.junit.Assert.*; + +public class UniqueIndexCleanerTest { + private NodeBuilder builder = EMPTY_NODE.builder(); + + @Test + public void nothingCleaned() throws Exception{ + builder.child("a").setProperty(PROP_CREATED, 100); + builder.child("b").setProperty(PROP_CREATED, 100); + + refresh(); + + UniqueIndexCleaner cleaner = new UniqueIndexCleaner(MILLISECONDS, 1); + cleaner.clean(builder, 10); + assertThat(copyOf(builder.getChildNodeNames()), containsInAnyOrder("a", "b")); + } + + @Test + public void cleanWithMargin() throws Exception{ + builder.child("a").setProperty(PROP_CREATED, 100); + builder.child("b").setProperty(PROP_CREATED, 200); + + refresh(); + UniqueIndexCleaner cleaner = new UniqueIndexCleaner(MILLISECONDS, 100); + cleaner.clean(builder, 200); + assertThat(copyOf(builder.getChildNodeNames()), containsInAnyOrder("b")); + } + + private void refresh(){ + builder = builder.getNodeState().builder(); + } +} \ No newline at end of file diff --git a/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidatorTest.java b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidatorTest.java new file mode 100644 index 0000000000..717086e33f --- /dev/null +++ b/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/UniquenessConstraintValidatorTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.jackrabbit.oak.plugins.index.lucene.property; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyDefinition; +import org.apache.jackrabbit.oak.plugins.index.lucene.PropertyUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.junit.Test; + +import static java.util.Collections.singletonList; +import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT; +import static org.apache.jackrabbit.oak.api.CommitFailedException.CONSTRAINT; +import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE; +import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; + +public class UniquenessConstraintValidatorTest { + private NodeState root = INITIAL_CONTENT; + private NodeBuilder builder = EMPTY_NODE.builder(); + private IndexDefinitionBuilder defnb = new IndexDefinitionBuilder(); + private String indexPath = "/oak:index/foo"; + + @Test + public void singleUniqueProperty() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + PropertyUpdateCallback callback = newCallback(); + + callback.propertyUpdated("/a", "foo", pd("foo"), + null, createProperty("foo", "bar")); + callback.propertyUpdated("/b", "foo", pd("foo"), + null, createProperty("foo", "bar")); + + try { + callback.done(); + fail(); + } catch (CommitFailedException e) { + assertEquals(CONSTRAINT, e.getType()); + assertEquals(30, e.getCode()); + + assertThat(e.getMessage(), containsString(indexPath)); + assertThat(e.getMessage(), containsString("/a")); + assertThat(e.getMessage(), containsString("/b")); + assertThat(e.getMessage(), containsString("foo")); + } + } + + @Test + public void multipleUniqueProperties() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + defnb.indexRule("nt:base").property("foo2").unique(); + + PropertyUpdateCallback callback = newCallback(); + + propertyUpdated(callback, "/a", "foo", "bar"); + propertyUpdated(callback, "/a", "foo2", "bar"); + + //As properties are different this should pass + callback.done(); + } + + @Test(expected = CommitFailedException.class) + public void firstStore_PreExist() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + PropertyUpdateCallback callback = newCallback(); + propertyUpdated(callback, "/a", "foo", "bar"); + + builder = builder.getNodeState().builder(); + + callback = newCallback(); + propertyUpdated(callback, "/b", "foo", "bar"); + callback.done(); + } + + @Test + public void secondStore_SamePath() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + PropertyIndexUpdateCallback callback = newCallback(); + propertyUpdated(callback, "/a", "foo", "bar"); + + callback.getUniquenessConstraintValidator() + .setSecondStore((propertyRelativePath, value) -> singletonList("/a")); + + //Should work as paths for unique property are same + callback.done(); + } + + @Test(expected = CommitFailedException.class) + public void secondStore_DiffPath() throws Exception{ + defnb.indexRule("nt:base").property("foo").unique(); + + PropertyIndexUpdateCallback callback = newCallback(); + propertyUpdated(callback, "/a", "foo", "bar"); + + callback.getUniquenessConstraintValidator() + .setSecondStore((propertyRelativePath, value) -> singletonList("/b")); + + callback.done(); + } + + private void propertyUpdated(PropertyUpdateCallback callback, String nodePath, String propertyName, String value){ + callback.propertyUpdated(nodePath, propertyName, pd(propertyName), + null, createProperty(propertyName, value)); + } + + private PropertyIndexUpdateCallback newCallback(){ + return new PropertyIndexUpdateCallback(indexPath, builder); + } + + private PropertyDefinition pd(String propName){ + IndexDefinition defn = new IndexDefinition(root, defnb.build(), indexPath); + return defn.getApplicableIndexingRule("nt:base").getConfig(propName); + } +} \ No newline at end of file