diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java index f1232cc139..058bdd6334 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java @@ -23,13 +23,20 @@ import javax.annotation.CheckForNull; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.mount.Mount; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.commons.hash.SipHash; +import java.util.HashMap; +import java.util.Map; + /** * An approximate descendant node counter mechanism. */ @@ -48,16 +55,40 @@ public class NodeCounterEditor implements Editor { private final NodeCounterRoot root; private final NodeCounterEditor parent; private final String name; - private long countOffset; + private final MountInfoProvider mountInfoProvider; + private final Map countOffsets; + private final Mount currentMount; + private final boolean mountCanChange; private SipHash hash; - - public NodeCounterEditor(NodeCounterRoot root, NodeCounterEditor parent, String name, SipHash hash) { + + + NodeCounterEditor(NodeCounterRoot root, MountInfoProvider mountInfoProvider) { + this.root = root; + this.name = "/"; + this.parent = null; + this.mountInfoProvider = mountInfoProvider; + this.currentMount = mountInfoProvider.getDefaultMount(); + this.mountCanChange = true; + this.countOffsets = new HashMap<>(); + } + + private NodeCounterEditor(NodeCounterRoot root, NodeCounterEditor parent, String name, SipHash hash, MountInfoProvider mountInfoProvider) { this.parent = parent; this.root = root; this.name = name; this.hash = hash; - } - + this.mountInfoProvider = mountInfoProvider; + this.countOffsets = new HashMap<>(); + if (parent.mountCanChange) { + String path = getPath(); + this.currentMount = mountInfoProvider.getMountByPath(path); + this.mountCanChange = currentMount.isDefault() && supportMounts(path); + } else { + this.currentMount = this.parent.currentMount; + this.mountCanChange = false; + } + } + private SipHash getHash() { if (hash != null) { return hash; @@ -90,59 +121,78 @@ public class NodeCounterEditor implements Editor { private void leaveOld(NodeState before, NodeState after) throws CommitFailedException { - long offset = ApproximateCounter.calculateOffset( - countOffset, root.resolution); - if (offset == 0) { + if (countOffsets.isEmpty()) { return; } - // only read the value of the property if really needed - NodeBuilder builder = getBuilder(); - PropertyState p = builder.getProperty(COUNT_PROPERTY_NAME); - long count = p == null ? 0 : p.getValue(Type.LONG); - offset = ApproximateCounter.adjustOffset(count, - offset, root.resolution); - if (offset == 0) { - return; - } - count += offset; - root.callback.indexUpdate(); - if (count == 0) { - if (builder.getChildNodeCount(1) >= 0) { - builder.removeProperty(COUNT_PROPERTY_NAME); + boolean updated = false; + for (Map.Entry e : countOffsets.entrySet()) { + long offset = ApproximateCounter.calculateOffset(e.getValue(), root.resolution); + if (offset == 0) { + continue; + } + // only read the value of the property if really needed + NodeBuilder builder = getBuilder(e.getKey()); + PropertyState p = builder.getProperty(COUNT_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + offset = ApproximateCounter.adjustOffset(count, offset, root.resolution); + if (offset == 0) { + continue; + } + updated = true; + count += offset; + if (count == 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_PROPERTY_NAME); + } else { + builder.remove(); + } } else { - builder.remove(); + builder.setProperty(COUNT_PROPERTY_NAME, count); } - } else { - builder.setProperty(COUNT_PROPERTY_NAME, count); + } + if (updated) { + root.callback.indexUpdate(); } } - - public void leaveNew(NodeState before, NodeState after) - throws CommitFailedException { - if (countOffset == 0) { + + public void leaveNew(NodeState before, NodeState after) throws CommitFailedException { + if (countOffsets.isEmpty()) { return; } - NodeBuilder builder = getBuilder(); - PropertyState p = builder.getProperty(COUNT_HASH_PROPERTY_NAME); - long count = p == null ? 0 : p.getValue(Type.LONG); - count += countOffset; root.callback.indexUpdate(); - if (count <= 0) { - if (builder.getChildNodeCount(1) >= 0) { - builder.removeProperty(COUNT_HASH_PROPERTY_NAME); + for (Map.Entry e : countOffsets.entrySet()) { + NodeBuilder builder = getBuilder(e.getKey()); + int countOffset = e.getValue(); + + PropertyState p = builder.getProperty(COUNT_HASH_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + count += countOffset; + if (count <= 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_HASH_PROPERTY_NAME); + } else { + builder.remove(); + } } else { - builder.remove(); + builder.setProperty(COUNT_HASH_PROPERTY_NAME, count); } + } + } + + private NodeBuilder getBuilder(Mount mount) { + if (parent == null) { + return root.definition.child(Multiplexers.getNodeForMount(mount, DATA_NODE_NAME)); } else { - builder.setProperty(COUNT_HASH_PROPERTY_NAME, count); + return parent.getBuilder(mount).child(name); } } - private NodeBuilder getBuilder() { + private String getPath() { if (parent == null) { - return root.definition.child(DATA_NODE_NAME); + return name; + } else { + return PathUtils.concat(parent.getPath(), name); } - return parent.getBuilder().child(name); } @Override @@ -178,11 +228,11 @@ public class NodeCounterEditor implements Editor { // with bitMask=1024: with a probability of 1:1024, if ((h.hashCode() & root.bitMask) == 0) { // add 1024 - count(root.bitMask + 1); + count(root.bitMask + 1, currentMount); } - return getChildIndexEditor(name, h); + return getChildIndexEditor(name, h); } - count(1); + count(1, currentMount); return getChildIndexEditor(name, null); } @@ -194,26 +244,33 @@ public class NodeCounterEditor implements Editor { SipHash h = new SipHash(getHash(), name.hashCode()); // with bitMask=1024: with a probability of 1:1024, if ((h.hashCode() & root.bitMask) == 0) { - // subtract 1024 - count(-(root.bitMask + 1)); + // subtract 1024 + count(-(root.bitMask + 1), currentMount); } return getChildIndexEditor(name, h); } - count(-1); + count(-1, currentMount); return getChildIndexEditor(name, null); } - private void count(int offset) { - countOffset += offset; + private void count(int offset, Mount mount) { + countOffsets.compute(mount, (m, v) -> v == null ? offset : v + offset); if (parent != null) { - parent.count(offset); + parent.count(offset, mount); } } private Editor getChildIndexEditor(String name, SipHash hash) { - return new NodeCounterEditor(root, this, name, hash); + return new NodeCounterEditor(root, this, name, hash, mountInfoProvider); } - + + private boolean supportMounts(String path) { + return mountInfoProvider + .getNonDefaultMounts() + .stream() + .anyMatch(m -> m.isSupportFragmentUnder(path) || m.isUnder(path)); + } + public static class NodeCounterRoot { final int resolution; final long seed; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java index 09aca624ff..2fa69e4e23 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java @@ -24,6 +24,7 @@ import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import org.apache.felix.scr.annotations.Component; +import org.apache.felix.scr.annotations.Reference; import org.apache.felix.scr.annotations.Service; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; @@ -33,6 +34,8 @@ import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditor.NodeCounterRoot; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; @@ -46,6 +49,9 @@ public class NodeCounterEditorProvider implements IndexEditorProvider { public static final String SEED = "seed"; + @Reference + private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); + @Override @CheckForNull public Editor getIndexEditor(@Nonnull String type, @@ -75,7 +81,11 @@ public class NodeCounterEditorProvider implements IndexEditorProvider { } NodeCounterRoot rootData = new NodeCounterRoot( resolution, seed, definition, root, callback); - return new NodeCounterEditor(rootData, null, "/", null); + return new NodeCounterEditor(rootData, mountInfoProvider); } + public NodeCounterEditorProvider with(MountInfoProvider mountInfoProvider) { + this.mountInfoProvider = mountInfoProvider; + return this; + } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java index 2d698aca80..5a429c02ac 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java @@ -21,6 +21,9 @@ package org.apache.jackrabbit.oak.plugins.index.counter.jmx; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Objects; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; @@ -28,6 +31,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditor; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; @@ -124,36 +128,22 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB // no index return -1; } - s = child(s, NodeCounterEditor.DATA_NODE_NAME); - if (!s.exists()) { + + if (!dataNodeExists(s)) { // no index data (not yet indexed, or very few nodes) return -1; - } - s = child(s, PathUtils.elements(path)); - if (s == null || !s.exists()) { - // we have an index, but no data - long x = 0; - if (max) { - // in the index, the resolution is lower - x += ApproximateCounter.COUNT_RESOLUTION * 20; - } - return x; } - p = s.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME); - if (p == null) { - // we have an index, but no data - long x = 0; - if (max) { - // in the index, the resolution is lower - x += ApproximateCounter.COUNT_RESOLUTION * 20; - } - return x; + + long sum = getIndexingData(s, path) + .map(n -> n.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME)) + .filter(Objects::nonNull) + .mapToLong(v -> v.getValue(Type.LONG)) + .sum(); + if (sum == 0) { + return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; + } else { + return sum + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); } - long x = p.getValue(Type.LONG); - if (max) { - x += ApproximateCounter.COUNT_RESOLUTION; - } - return x; } private static long getCombinedCount(NodeState root, String path, NodeState s, boolean max) { @@ -169,20 +159,22 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB // no index return -1; } - s = child(s, NodeCounterEditor.DATA_NODE_NAME); - if (!s.exists()) { + + if (!dataNodeExists(s)) { // no index data (not yet indexed, or very few nodes) return -1; } - s = child(s, PathUtils.elements(path)); - if (s != null && s.exists()) { - value = getCombinedCountIfAvailable(s); - if (value != null) { - return value + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); - } + + long sum = getIndexingData(s, path) + .map(NodeCounter::getCombinedCountIfAvailable) + .filter(Objects::nonNull) + .mapToLong(Long::longValue) + .sum(); + if (sum == 0) { + return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; + } else { + return sum + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); } - // we have an index, but no data - return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; } private static Long getCombinedCountIfAvailable(NodeState s) { @@ -236,4 +228,27 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB } } + private static Stream getIndexingData(NodeState indexDefinition, String path) { + Iterable pathElements = PathUtils.elements(path); + + return StreamSupport.stream(indexDefinition.getChildNodeEntries().spliterator(), false) + .filter(NodeCounter::isDataNodeName) + .map(ChildNodeEntry::getNodeState) + .map(n -> child(n, pathElements)) + .filter(Objects::nonNull) + .filter(NodeState::exists); + } + + private static boolean isDataNodeName(ChildNodeEntry childNodeEntry) { + String name = childNodeEntry.getName(); + return NodeCounterEditor.DATA_NODE_NAME.equals(name) + || (name.startsWith(":") && name.endsWith("-" + Multiplexers.stripStartingColon(NodeCounterEditor.DATA_NODE_NAME))); + } + + private static boolean dataNodeExists(NodeState indexDefinition) { + return StreamSupport + .stream(indexDefinition.getChildNodeEntries().spliterator(), false) + .anyMatch(NodeCounter::isDataNodeName); + } + } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java index f4476ae98a..a70c542392 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java @@ -165,7 +165,7 @@ public class Multiplexers { return "-" + stripStartingColon(name); } - private static String stripStartingColon(String name) { + public static String stripStartingColon(String name) { if (name.startsWith(":")) { return name.substring(1); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java new file mode 100644 index 0000000000..c43d992ef5 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.counter; + +import com.google.common.base.Predicate; +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.ContentSession; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Root; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.mount.Mount; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class MountsNodeCounterTest { + + private NodeStore nodeStore; + + private Root root; + + private MountInfoProvider mip; + + private Whiteboard wb; + + @Before + public void before() throws Exception { + ContentSession session = createRepository().login(null, null); + root = session.getLatestRoot(); + } + + @Test + public void testMultipleMounts() throws CommitFailedException { + root.getTree("/oak:index/counter").setProperty("resolution", 1); + root.commit(); + + Tree rootTree = root.getTree("/"); + + Tree apps = rootTree.addChild("apps"); + Tree libs = rootTree.addChild("libs"); + Tree content = rootTree.addChild("content"); + Tree nested = rootTree.addChild("nested"); + Tree nestedMount = nested.addChild("mount"); + Tree fragments = rootTree.addChild("var").addChild("fragments").addChild("oak:mount-libs"); + + addChildren(apps, 100); + addChildren(libs, 200); + addChildren(content, 400); + addChildren(nested, 800); + addChildren(nestedMount, 1600); + addChildren(fragments, 3200); + + root.commit(); + runAsyncIndex(); + + // leaves: + Mount defaultMount = mip.getDefaultMount(); + Mount libsMount = mip.getMountByName("libs"); + + assertCountEquals(100, libsMount, "apps"); + assertCountEquals(200, libsMount, "libs"); + assertCountEquals(400, defaultMount, "content"); + assertCountEquals(800, defaultMount, "nested"); + assertCountEquals(1600, libsMount, "nested/mount"); + assertCountEquals(3200, libsMount, "var"); + assertCountEquals(3200, libsMount, "var/fragments"); + assertCountEquals(3200, libsMount, "var/fragments/oak:mount-libs"); + assertCountEquals(0, defaultMount, "var"); + assertCountEquals(0, defaultMount, "var/fragments"); + + assertCountEquals(100 + 200 + 1600 + 3200, libsMount, ""); + assertCountEquals(1600, libsMount, "nested"); + } + + private void assertCountEquals(int expectedCount, Mount mount, String path) { + String p = PathUtils.concat("/oak:index/counter", Multiplexers.getNodeForMount(mount, ":index"), path); + NodeState s = nodeStore.getRoot(); + for (String element : PathUtils.elements(p)) { + s = s.getChildNode(element); + if (s == null) { + if (expectedCount == 0) { + return; + } + fail("Can't find node " + p); + } + } + PropertyState ps = s.getProperty(":cnt"); + if (ps == null) { + if (expectedCount == 0) { + return; + } + fail("There's no :cnt property on " + p); + } + long v = ps.getValue(Type.LONG); + + + assertTrue("expected:<" + expectedCount + "> but was:<" + v + ">", Math.abs(expectedCount - v) < 10); + } + + private static void addChildren(Tree tree, int count) { + for (int i = 0; i < count; i++) { + tree.addChild("n-" + i); + } + } + + protected ContentRepository createRepository() { + Mounts.Builder builder = Mounts.newBuilder(); + builder.mount("libs", false, Arrays.asList("/var/fragments"), Arrays.asList("/apps", "/libs", "/nested/mount")); + mip = builder.build(); + + nodeStore = new MemoryNodeStore(); + Oak oak = new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with(new PropertyIndexEditorProvider().with(mip)) + .with(new NodeCounterEditorProvider().with(mip)) + //Effectively disable async indexing auto run + //such that we can control run timing as per test requirement + .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); + + wb = oak.getWhiteboard(); + return oak.createContentRepository(); + } + + private void runAsyncIndex() { + Runnable async = WhiteboardUtils.getService(wb, Runnable.class, new Predicate() { + @Override + public boolean apply(@Nullable Runnable input) { + return input instanceof AsyncIndexUpdate; + } + }); + assertNotNull(async); + async.run(); + root.refresh(); + } + +}