diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java index 5ce0e1174f..9186172658 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/Oak.java @@ -76,6 +76,7 @@ import org.apache.jackrabbit.oak.plugins.index.IndexMBeanRegistration; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateProvider; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounterMBean; +import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounterOld; import org.apache.jackrabbit.oak.plugins.index.property.jmx.PropertyIndexAsyncReindex; import org.apache.jackrabbit.oak.plugins.index.property.jmx.PropertyIndexAsyncReindexMBean; import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; @@ -694,8 +695,13 @@ public class Oak { PropertyIndexAsyncReindexMBean.TYPE, "async")); } - regs.add(registerMBean(whiteboard, NodeCounterMBean.class, - new NodeCounter(store), NodeCounterMBean.TYPE, "nodeCounter")); + if (NodeCounter.USE_OLD_COUNTER) { + regs.add(registerMBean(whiteboard, NodeCounterMBean.class, + new NodeCounterOld(store), NodeCounterMBean.TYPE, "nodeCounter")); + } else { + regs.add(registerMBean(whiteboard, NodeCounterMBean.class, + new NodeCounter(store), NodeCounterMBean.TYPE, "nodeCounter")); + } regs.add(registerMBean(whiteboard, QueryEngineSettingsMBean.class, queryEngineSettings, QueryEngineSettingsMBean.TYPE, "settings")); diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java index f1232cc139..058bdd6334 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditor.java @@ -23,13 +23,20 @@ import javax.annotation.CheckForNull; import org.apache.jackrabbit.oak.api.CommitFailedException; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.mount.Mount; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.commons.hash.SipHash; +import java.util.HashMap; +import java.util.Map; + /** * An approximate descendant node counter mechanism. */ @@ -48,16 +55,40 @@ public class NodeCounterEditor implements Editor { private final NodeCounterRoot root; private final NodeCounterEditor parent; private final String name; - private long countOffset; + private final MountInfoProvider mountInfoProvider; + private final Map countOffsets; + private final Mount currentMount; + private final boolean mountCanChange; private SipHash hash; - - public NodeCounterEditor(NodeCounterRoot root, NodeCounterEditor parent, String name, SipHash hash) { + + + NodeCounterEditor(NodeCounterRoot root, MountInfoProvider mountInfoProvider) { + this.root = root; + this.name = "/"; + this.parent = null; + this.mountInfoProvider = mountInfoProvider; + this.currentMount = mountInfoProvider.getDefaultMount(); + this.mountCanChange = true; + this.countOffsets = new HashMap<>(); + } + + private NodeCounterEditor(NodeCounterRoot root, NodeCounterEditor parent, String name, SipHash hash, MountInfoProvider mountInfoProvider) { this.parent = parent; this.root = root; this.name = name; this.hash = hash; - } - + this.mountInfoProvider = mountInfoProvider; + this.countOffsets = new HashMap<>(); + if (parent.mountCanChange) { + String path = getPath(); + this.currentMount = mountInfoProvider.getMountByPath(path); + this.mountCanChange = currentMount.isDefault() && supportMounts(path); + } else { + this.currentMount = this.parent.currentMount; + this.mountCanChange = false; + } + } + private SipHash getHash() { if (hash != null) { return hash; @@ -90,59 +121,78 @@ public class NodeCounterEditor implements Editor { private void leaveOld(NodeState before, NodeState after) throws CommitFailedException { - long offset = ApproximateCounter.calculateOffset( - countOffset, root.resolution); - if (offset == 0) { + if (countOffsets.isEmpty()) { return; } - // only read the value of the property if really needed - NodeBuilder builder = getBuilder(); - PropertyState p = builder.getProperty(COUNT_PROPERTY_NAME); - long count = p == null ? 0 : p.getValue(Type.LONG); - offset = ApproximateCounter.adjustOffset(count, - offset, root.resolution); - if (offset == 0) { - return; - } - count += offset; - root.callback.indexUpdate(); - if (count == 0) { - if (builder.getChildNodeCount(1) >= 0) { - builder.removeProperty(COUNT_PROPERTY_NAME); + boolean updated = false; + for (Map.Entry e : countOffsets.entrySet()) { + long offset = ApproximateCounter.calculateOffset(e.getValue(), root.resolution); + if (offset == 0) { + continue; + } + // only read the value of the property if really needed + NodeBuilder builder = getBuilder(e.getKey()); + PropertyState p = builder.getProperty(COUNT_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + offset = ApproximateCounter.adjustOffset(count, offset, root.resolution); + if (offset == 0) { + continue; + } + updated = true; + count += offset; + if (count == 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_PROPERTY_NAME); + } else { + builder.remove(); + } } else { - builder.remove(); + builder.setProperty(COUNT_PROPERTY_NAME, count); } - } else { - builder.setProperty(COUNT_PROPERTY_NAME, count); + } + if (updated) { + root.callback.indexUpdate(); } } - - public void leaveNew(NodeState before, NodeState after) - throws CommitFailedException { - if (countOffset == 0) { + + public void leaveNew(NodeState before, NodeState after) throws CommitFailedException { + if (countOffsets.isEmpty()) { return; } - NodeBuilder builder = getBuilder(); - PropertyState p = builder.getProperty(COUNT_HASH_PROPERTY_NAME); - long count = p == null ? 0 : p.getValue(Type.LONG); - count += countOffset; root.callback.indexUpdate(); - if (count <= 0) { - if (builder.getChildNodeCount(1) >= 0) { - builder.removeProperty(COUNT_HASH_PROPERTY_NAME); + for (Map.Entry e : countOffsets.entrySet()) { + NodeBuilder builder = getBuilder(e.getKey()); + int countOffset = e.getValue(); + + PropertyState p = builder.getProperty(COUNT_HASH_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + count += countOffset; + if (count <= 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_HASH_PROPERTY_NAME); + } else { + builder.remove(); + } } else { - builder.remove(); + builder.setProperty(COUNT_HASH_PROPERTY_NAME, count); } + } + } + + private NodeBuilder getBuilder(Mount mount) { + if (parent == null) { + return root.definition.child(Multiplexers.getNodeForMount(mount, DATA_NODE_NAME)); } else { - builder.setProperty(COUNT_HASH_PROPERTY_NAME, count); + return parent.getBuilder(mount).child(name); } } - private NodeBuilder getBuilder() { + private String getPath() { if (parent == null) { - return root.definition.child(DATA_NODE_NAME); + return name; + } else { + return PathUtils.concat(parent.getPath(), name); } - return parent.getBuilder().child(name); } @Override @@ -178,11 +228,11 @@ public class NodeCounterEditor implements Editor { // with bitMask=1024: with a probability of 1:1024, if ((h.hashCode() & root.bitMask) == 0) { // add 1024 - count(root.bitMask + 1); + count(root.bitMask + 1, currentMount); } - return getChildIndexEditor(name, h); + return getChildIndexEditor(name, h); } - count(1); + count(1, currentMount); return getChildIndexEditor(name, null); } @@ -194,26 +244,33 @@ public class NodeCounterEditor implements Editor { SipHash h = new SipHash(getHash(), name.hashCode()); // with bitMask=1024: with a probability of 1:1024, if ((h.hashCode() & root.bitMask) == 0) { - // subtract 1024 - count(-(root.bitMask + 1)); + // subtract 1024 + count(-(root.bitMask + 1), currentMount); } return getChildIndexEditor(name, h); } - count(-1); + count(-1, currentMount); return getChildIndexEditor(name, null); } - private void count(int offset) { - countOffset += offset; + private void count(int offset, Mount mount) { + countOffsets.compute(mount, (m, v) -> v == null ? offset : v + offset); if (parent != null) { - parent.count(offset); + parent.count(offset, mount); } } private Editor getChildIndexEditor(String name, SipHash hash) { - return new NodeCounterEditor(root, this, name, hash); + return new NodeCounterEditor(root, this, name, hash, mountInfoProvider); } - + + private boolean supportMounts(String path) { + return mountInfoProvider + .getNonDefaultMounts() + .stream() + .anyMatch(m -> m.isSupportFragmentUnder(path) || m.isUnder(path)); + } + public static class NodeCounterRoot { final int resolution; final long seed; diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorOld.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorOld.java new file mode 100644 index 0000000000..ceef24a8e3 --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorOld.java @@ -0,0 +1,237 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.counter; + +import javax.annotation.CheckForNull; + +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; +import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; +import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.state.NodeBuilder; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.commons.hash.SipHash; + +/** + * An approximate descendant node counter mechanism. + */ +@Deprecated +public class NodeCounterEditorOld implements Editor { + + public static final String DATA_NODE_NAME = ":index"; + + // the property that is used with the "old" (pseudo-random number generator based) method + public static final String COUNT_PROPERTY_NAME = ":count"; + + // the property that is used with the "new" (hash of the path based) method + public static final String COUNT_HASH_PROPERTY_NAME = ":cnt"; + + public static final int DEFAULT_RESOLUTION = 1000; + + private final NodeCounterRoot root; + private final NodeCounterEditorOld parent; + private final String name; + private long countOffset; + private SipHash hash; + + public NodeCounterEditorOld(NodeCounterRoot root, NodeCounterEditorOld parent, String name, SipHash hash) { + this.parent = parent; + this.root = root; + this.name = name; + this.hash = hash; + } + + private SipHash getHash() { + if (hash != null) { + return hash; + } + SipHash h; + if (parent == null) { + h = new SipHash(root.seed); + } else { + h = new SipHash(parent.getHash(), name.hashCode()); + } + this.hash = h; + return h; + } + + @Override + public void enter(NodeState before, NodeState after) + throws CommitFailedException { + // nothing to do + } + + @Override + public void leave(NodeState before, NodeState after) + throws CommitFailedException { + if (NodeCounter.COUNT_HASH) { + leaveNew(before, after); + return; + } + leaveOld(before, after); + } + + private void leaveOld(NodeState before, NodeState after) + throws CommitFailedException { + long offset = ApproximateCounter.calculateOffset( + countOffset, root.resolution); + if (offset == 0) { + return; + } + // only read the value of the property if really needed + NodeBuilder builder = getBuilder(); + PropertyState p = builder.getProperty(COUNT_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + offset = ApproximateCounter.adjustOffset(count, + offset, root.resolution); + if (offset == 0) { + return; + } + count += offset; + root.callback.indexUpdate(); + if (count == 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_PROPERTY_NAME); + } else { + builder.remove(); + } + } else { + builder.setProperty(COUNT_PROPERTY_NAME, count); + } + } + + public void leaveNew(NodeState before, NodeState after) + throws CommitFailedException { + if (countOffset == 0) { + return; + } + NodeBuilder builder = getBuilder(); + PropertyState p = builder.getProperty(COUNT_HASH_PROPERTY_NAME); + long count = p == null ? 0 : p.getValue(Type.LONG); + count += countOffset; + root.callback.indexUpdate(); + if (count <= 0) { + if (builder.getChildNodeCount(1) >= 0) { + builder.removeProperty(COUNT_HASH_PROPERTY_NAME); + } else { + builder.remove(); + } + } else { + builder.setProperty(COUNT_HASH_PROPERTY_NAME, count); + } + } + + private NodeBuilder getBuilder() { + if (parent == null) { + return root.definition.child(DATA_NODE_NAME); + } + return parent.getBuilder().child(name); + } + + @Override + public void propertyAdded(PropertyState after) throws CommitFailedException { + // nothing to do + } + + @Override + public void propertyChanged(PropertyState before, PropertyState after) + throws CommitFailedException { + // nothing to do + } + + @Override + public void propertyDeleted(PropertyState before) + throws CommitFailedException { + // nothing to do + } + + @Override + @CheckForNull + public Editor childNodeChanged(String name, NodeState before, NodeState after) + throws CommitFailedException { + return getChildIndexEditor(name, null); + } + + @Override + @CheckForNull + public Editor childNodeAdded(String name, NodeState after) + throws CommitFailedException { + if (NodeCounter.COUNT_HASH) { + SipHash h = new SipHash(getHash(), name.hashCode()); + // with bitMask=1024: with a probability of 1:1024, + if ((h.hashCode() & root.bitMask) == 0) { + // add 1024 + count(root.bitMask + 1); + } + return getChildIndexEditor(name, h); + } + count(1); + return getChildIndexEditor(name, null); + } + + @Override + @CheckForNull + public Editor childNodeDeleted(String name, NodeState before) + throws CommitFailedException { + if (NodeCounter.COUNT_HASH) { + SipHash h = new SipHash(getHash(), name.hashCode()); + // with bitMask=1024: with a probability of 1:1024, + if ((h.hashCode() & root.bitMask) == 0) { + // subtract 1024 + count(-(root.bitMask + 1)); + } + return getChildIndexEditor(name, h); + } + count(-1); + return getChildIndexEditor(name, null); + } + + private void count(int offset) { + countOffset += offset; + if (parent != null) { + parent.count(offset); + } + } + + private Editor getChildIndexEditor(String name, SipHash hash) { + return new NodeCounterEditorOld(root, this, name, hash); + } + + public static class NodeCounterRoot { + final int resolution; + final long seed; + final int bitMask; + final NodeBuilder definition; + final NodeState root; + final IndexUpdateCallback callback; + + NodeCounterRoot(int resolution, long seed, NodeBuilder definition, NodeState root, IndexUpdateCallback callback) { + this.resolution = resolution; + this.seed = seed; + // if resolution is 1000, then the bitMask is 1023 (bits 0..9 set) + this.bitMask = (Integer.highestOneBit(resolution) * 2) - 1; + this.definition = definition; + this.root = root; + this.callback = callback; + } + } + +} \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java index 2f43e5d155..f9d5d7c185 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/NodeCounterEditorProvider.java @@ -28,12 +28,14 @@ import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider; import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback; -import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditor.NodeCounterRoot; import org.apache.jackrabbit.oak.plugins.index.counter.jmx.NodeCounter; import org.apache.jackrabbit.oak.spi.commit.Editor; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; import org.apache.jackrabbit.oak.spi.state.NodeBuilder; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.osgi.service.component.annotations.Component; +import org.osgi.service.component.annotations.Reference; @Component(service = IndexEditorProvider.class) public class NodeCounterEditorProvider implements IndexEditorProvider { @@ -44,6 +46,9 @@ public class NodeCounterEditorProvider implements IndexEditorProvider { public static final String SEED = "seed"; + @Reference + private MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider(); + @Override @CheckForNull public Editor getIndexEditor(@Nonnull String type, @@ -71,9 +76,20 @@ public class NodeCounterEditorProvider implements IndexEditorProvider { definition.setProperty(SEED, seed); } } - NodeCounterRoot rootData = new NodeCounterRoot( - resolution, seed, definition, root, callback); - return new NodeCounterEditor(rootData, null, "/", null); + + if (NodeCounter.USE_OLD_COUNTER) { + NodeCounterEditorOld.NodeCounterRoot rootData = new NodeCounterEditorOld.NodeCounterRoot( + resolution, seed, definition, root, callback); + return new NodeCounterEditorOld(rootData, null, "/", null); + } else { + NodeCounterEditor.NodeCounterRoot rootData = new NodeCounterEditor.NodeCounterRoot( + resolution, seed, definition, root, callback); + return new NodeCounterEditor(rootData, mountInfoProvider); + } } + public NodeCounterEditorProvider with(MountInfoProvider mountInfoProvider) { + this.mountInfoProvider = mountInfoProvider; + return this; + } } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java index 2d698aca80..ad07750b13 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounter.java @@ -21,6 +21,9 @@ package org.apache.jackrabbit.oak.plugins.index.counter.jmx; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Objects; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Type; @@ -28,6 +31,7 @@ import org.apache.jackrabbit.oak.commons.PathUtils; import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean; import org.apache.jackrabbit.oak.plugins.index.IndexConstants; import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditor; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; import org.apache.jackrabbit.oak.spi.state.NodeState; import org.apache.jackrabbit.oak.spi.state.NodeStore; @@ -45,6 +49,8 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB public final static boolean COUNT_HASH = Boolean.parseBoolean(System.getProperty("oak.countHashed", "true")); + public static final boolean USE_OLD_COUNTER = Boolean.getBoolean("oak.index.useCounterOld"); + private final NodeStore store; public NodeCounter(NodeStore store) { @@ -85,6 +91,14 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB * (maximum) estimated number of descendant nodes */ public static long getEstimatedNodeCount(NodeState root, String path, boolean max) { + if (USE_OLD_COUNTER) { + return NodeCounterOld.getEstimatedNodeCount(root, path, max); + } else { + return doGetEstimatedNodeCount(root, path, max); + } + } + + private static long doGetEstimatedNodeCount(NodeState root, String path, boolean max) { // check if there is a property in the node itself // (for property index nodes) NodeState s = child(root, @@ -124,36 +138,22 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB // no index return -1; } - s = child(s, NodeCounterEditor.DATA_NODE_NAME); - if (!s.exists()) { + + if (!dataNodeExists(s)) { // no index data (not yet indexed, or very few nodes) return -1; - } - s = child(s, PathUtils.elements(path)); - if (s == null || !s.exists()) { - // we have an index, but no data - long x = 0; - if (max) { - // in the index, the resolution is lower - x += ApproximateCounter.COUNT_RESOLUTION * 20; - } - return x; } - p = s.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME); - if (p == null) { - // we have an index, but no data - long x = 0; - if (max) { - // in the index, the resolution is lower - x += ApproximateCounter.COUNT_RESOLUTION * 20; - } - return x; + + long sum = getIndexingData(s, path) + .map(n -> n.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME)) + .filter(Objects::nonNull) + .mapToLong(v -> v.getValue(Type.LONG)) + .sum(); + if (sum == 0) { + return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; + } else { + return sum + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); } - long x = p.getValue(Type.LONG); - if (max) { - x += ApproximateCounter.COUNT_RESOLUTION; - } - return x; } private static long getCombinedCount(NodeState root, String path, NodeState s, boolean max) { @@ -169,20 +169,22 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB // no index return -1; } - s = child(s, NodeCounterEditor.DATA_NODE_NAME); - if (!s.exists()) { + + if (!dataNodeExists(s)) { // no index data (not yet indexed, or very few nodes) return -1; } - s = child(s, PathUtils.elements(path)); - if (s != null && s.exists()) { - value = getCombinedCountIfAvailable(s); - if (value != null) { - return value + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); - } + + long sum = getIndexingData(s, path) + .map(NodeCounter::getCombinedCountIfAvailable) + .filter(Objects::nonNull) + .mapToLong(Long::longValue) + .sum(); + if (sum == 0) { + return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; + } else { + return sum + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); } - // we have an index, but no data - return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; } private static Long getCombinedCountIfAvailable(NodeState s) { @@ -236,4 +238,27 @@ public class NodeCounter extends AnnotatedStandardMBean implements NodeCounterMB } } + private static Stream getIndexingData(NodeState indexDefinition, String path) { + Iterable pathElements = PathUtils.elements(path); + + return StreamSupport.stream(indexDefinition.getChildNodeEntries().spliterator(), false) + .filter(NodeCounter::isDataNodeName) + .map(ChildNodeEntry::getNodeState) + .map(n -> child(n, pathElements)) + .filter(Objects::nonNull) + .filter(NodeState::exists); + } + + private static boolean isDataNodeName(ChildNodeEntry childNodeEntry) { + String name = childNodeEntry.getName(); + return NodeCounterEditor.DATA_NODE_NAME.equals(name) + || (name.startsWith(":") && name.endsWith("-" + Multiplexers.stripStartingColon(NodeCounterEditor.DATA_NODE_NAME))); + } + + private static boolean dataNodeExists(NodeState indexDefinition) { + return StreamSupport + .stream(indexDefinition.getChildNodeEntries().spliterator(), false) + .anyMatch(NodeCounter::isDataNodeName); + } + } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounterOld.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounterOld.java new file mode 100644 index 0000000000..e1414af236 --- /dev/null +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/counter/jmx/NodeCounterOld.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.counter.jmx; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; + +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean; +import org.apache.jackrabbit.oak.plugins.index.IndexConstants; +import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditor; +import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.plugins.index.counter.ApproximateCounter; + +/** + * A mechanism to retrieve node counter data. + */ +@Deprecated +public class NodeCounterOld extends AnnotatedStandardMBean implements NodeCounterMBean { + + /** + * Approximate count using the hashed name (deterministically, so that after + * adding a removing all nodes the count goes back to zero). + */ + public final static boolean COUNT_HASH = + Boolean.parseBoolean(System.getProperty("oak.countHashed", "true")); + + private final NodeStore store; + + public NodeCounterOld(NodeStore store) { + super(NodeCounterMBean.class); + this.store = store; + } + + private static NodeState child(NodeState n, String... path) { + return child(n, Arrays.asList(path)); + } + + private static NodeState child(NodeState n, Iterable path) { + for (String p : path) { + if (n == null) { + break; + } + if (p.length() > 0) { + n = n.getChildNode(p); + } + } + return n; + } + + @Override + public long getEstimatedNodeCount(String path) { + return getEstimatedNodeCount(store.getRoot(), path, false); + } + + /** + * Get the estimated number of nodes for a given path. + * + * @param root the root + * @param path the path + * @param max whether to get the maximum expected number of nodes (the + * stored value plus the resolution) + * @return -1 if unknown, 0 if the node does not exist (or, if max is false, + * if there are probably not many descendant nodes), or the + * (maximum) estimated number of descendant nodes + */ + public static long getEstimatedNodeCount(NodeState root, String path, boolean max) { + // check if there is a property in the node itself + // (for property index nodes) + NodeState s = child(root, + PathUtils.elements(path)); + if (s == null || !s.exists()) { + // node not found + return 0; + } + if (!max) { + long syncCount = ApproximateCounter.getCountSync(s); + if (syncCount != -1) { + return syncCount; + } + } + if (COUNT_HASH) { + return getCombinedCount(root, path, s, max); + } + return getEstimatedNodeCountOld(root, s, path, max); + } + + private static long getEstimatedNodeCountOld(NodeState root, NodeState s, String path, boolean max) { + // old code from here + PropertyState p = s.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME); + if (p != null) { + long x = p.getValue(Type.LONG); + if (max) { + // in the node itself, we just add the resolution + x += ApproximateCounter.COUNT_RESOLUTION; + } + return x; + } + // check in the counter index (if it exists) + s = child(root, + IndexConstants.INDEX_DEFINITIONS_NAME, + "counter"); + if (s == null || !s.exists()) { + // no index + return -1; + } + s = child(s, NodeCounterEditor.DATA_NODE_NAME); + if (!s.exists()) { + // no index data (not yet indexed, or very few nodes) + return -1; + } + s = child(s, PathUtils.elements(path)); + if (s == null || !s.exists()) { + // we have an index, but no data + long x = 0; + if (max) { + // in the index, the resolution is lower + x += ApproximateCounter.COUNT_RESOLUTION * 20; + } + return x; + } + p = s.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME); + if (p == null) { + // we have an index, but no data + long x = 0; + if (max) { + // in the index, the resolution is lower + x += ApproximateCounter.COUNT_RESOLUTION * 20; + } + return x; + } + long x = p.getValue(Type.LONG); + if (max) { + x += ApproximateCounter.COUNT_RESOLUTION; + } + return x; + } + + private static long getCombinedCount(NodeState root, String path, NodeState s, boolean max) { + Long value = getCombinedCountIfAvailable(s); + if (value != null) { + return value + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); + } + // check in the counter index (if it exists) + s = child(root, + IndexConstants.INDEX_DEFINITIONS_NAME, + "counter"); + if (s == null || !s.exists()) { + // no index + return -1; + } + s = child(s, NodeCounterEditor.DATA_NODE_NAME); + if (!s.exists()) { + // no index data (not yet indexed, or very few nodes) + return -1; + } + s = child(s, PathUtils.elements(path)); + if (s != null && s.exists()) { + value = getCombinedCountIfAvailable(s); + if (value != null) { + return value + (max ? ApproximateCounter.COUNT_RESOLUTION : 0); + } + } + // we have an index, but no data + return max ? ApproximateCounter.COUNT_RESOLUTION * 20 : 0; + } + + private static Long getCombinedCountIfAvailable(NodeState s) { + boolean found = false; + long x = 0; + PropertyState p = s.getProperty(NodeCounterEditor.COUNT_HASH_PROPERTY_NAME); + if (p != null) { + found = true; + x = p.getValue(Type.LONG); + } + p = s.getProperty(NodeCounterEditor.COUNT_PROPERTY_NAME); + if (p != null) { + found = true; + x += p.getValue(Type.LONG); + } + return found ? x : null; + } + + @Override + public String getEstimatedChildNodeCounts(String path, int level) { + StringBuilder buff = new StringBuilder(); + collectCounts(buff, path, level); + return buff.toString(); + } + + private void collectCounts(StringBuilder buff, String path, int level) { + long count = getEstimatedNodeCount(path); + if (count > 0) { + if (buff.length() > 0) { + buff.append(",\n"); + } + buff.append(path).append(": ").append(count); + } + if (level <= 0) { + return; + } + NodeState s = child(store.getRoot(), + PathUtils.elements(path)); + if (!s.exists()) { + return; + } + ArrayList names = new ArrayList(); + for (ChildNodeEntry c : s.getChildNodeEntries()) { + names.add(c.getName()); + } + Collections.sort(names); + for (String cn : names) { + s.getChildNode(cn); + String child = PathUtils.concat(path, cn); + collectCounts(buff, child, level - 1); + } + } + +} \ No newline at end of file diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java index f4476ae98a..a70c542392 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/property/Multiplexers.java @@ -165,7 +165,7 @@ public class Multiplexers { return "-" + stripStartingColon(name); } - private static String stripStartingColon(String name) { + public static String stripStartingColon(String name) { if (name.startsWith(":")) { return name.substring(1); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java new file mode 100644 index 0000000000..c43d992ef5 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/index/counter/MountsNodeCounterTest.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.plugins.index.counter; + +import com.google.common.base.Predicate; +import org.apache.jackrabbit.oak.InitialContent; +import org.apache.jackrabbit.oak.Oak; +import org.apache.jackrabbit.oak.api.CommitFailedException; +import org.apache.jackrabbit.oak.api.ContentRepository; +import org.apache.jackrabbit.oak.api.ContentSession; +import org.apache.jackrabbit.oak.api.PropertyState; +import org.apache.jackrabbit.oak.api.Root; +import org.apache.jackrabbit.oak.api.Tree; +import org.apache.jackrabbit.oak.api.Type; +import org.apache.jackrabbit.oak.commons.PathUtils; +import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate; +import org.apache.jackrabbit.oak.plugins.index.property.Multiplexers; +import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider; +import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore; +import org.apache.jackrabbit.oak.spi.mount.Mount; +import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider; +import org.apache.jackrabbit.oak.spi.mount.Mounts; +import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider; +import org.apache.jackrabbit.oak.spi.state.NodeState; +import org.apache.jackrabbit.oak.spi.state.NodeStore; +import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard; +import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils; +import org.junit.Before; +import org.junit.Test; + +import javax.annotation.Nullable; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class MountsNodeCounterTest { + + private NodeStore nodeStore; + + private Root root; + + private MountInfoProvider mip; + + private Whiteboard wb; + + @Before + public void before() throws Exception { + ContentSession session = createRepository().login(null, null); + root = session.getLatestRoot(); + } + + @Test + public void testMultipleMounts() throws CommitFailedException { + root.getTree("/oak:index/counter").setProperty("resolution", 1); + root.commit(); + + Tree rootTree = root.getTree("/"); + + Tree apps = rootTree.addChild("apps"); + Tree libs = rootTree.addChild("libs"); + Tree content = rootTree.addChild("content"); + Tree nested = rootTree.addChild("nested"); + Tree nestedMount = nested.addChild("mount"); + Tree fragments = rootTree.addChild("var").addChild("fragments").addChild("oak:mount-libs"); + + addChildren(apps, 100); + addChildren(libs, 200); + addChildren(content, 400); + addChildren(nested, 800); + addChildren(nestedMount, 1600); + addChildren(fragments, 3200); + + root.commit(); + runAsyncIndex(); + + // leaves: + Mount defaultMount = mip.getDefaultMount(); + Mount libsMount = mip.getMountByName("libs"); + + assertCountEquals(100, libsMount, "apps"); + assertCountEquals(200, libsMount, "libs"); + assertCountEquals(400, defaultMount, "content"); + assertCountEquals(800, defaultMount, "nested"); + assertCountEquals(1600, libsMount, "nested/mount"); + assertCountEquals(3200, libsMount, "var"); + assertCountEquals(3200, libsMount, "var/fragments"); + assertCountEquals(3200, libsMount, "var/fragments/oak:mount-libs"); + assertCountEquals(0, defaultMount, "var"); + assertCountEquals(0, defaultMount, "var/fragments"); + + assertCountEquals(100 + 200 + 1600 + 3200, libsMount, ""); + assertCountEquals(1600, libsMount, "nested"); + } + + private void assertCountEquals(int expectedCount, Mount mount, String path) { + String p = PathUtils.concat("/oak:index/counter", Multiplexers.getNodeForMount(mount, ":index"), path); + NodeState s = nodeStore.getRoot(); + for (String element : PathUtils.elements(p)) { + s = s.getChildNode(element); + if (s == null) { + if (expectedCount == 0) { + return; + } + fail("Can't find node " + p); + } + } + PropertyState ps = s.getProperty(":cnt"); + if (ps == null) { + if (expectedCount == 0) { + return; + } + fail("There's no :cnt property on " + p); + } + long v = ps.getValue(Type.LONG); + + + assertTrue("expected:<" + expectedCount + "> but was:<" + v + ">", Math.abs(expectedCount - v) < 10); + } + + private static void addChildren(Tree tree, int count) { + for (int i = 0; i < count; i++) { + tree.addChild("n-" + i); + } + } + + protected ContentRepository createRepository() { + Mounts.Builder builder = Mounts.newBuilder(); + builder.mount("libs", false, Arrays.asList("/var/fragments"), Arrays.asList("/apps", "/libs", "/nested/mount")); + mip = builder.build(); + + nodeStore = new MemoryNodeStore(); + Oak oak = new Oak(nodeStore) + .with(new InitialContent()) + .with(new OpenSecurityProvider()) + .with(new PropertyIndexEditorProvider().with(mip)) + .with(new NodeCounterEditorProvider().with(mip)) + //Effectively disable async indexing auto run + //such that we can control run timing as per test requirement + .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1)); + + wb = oak.getWhiteboard(); + return oak.createContentRepository(); + } + + private void runAsyncIndex() { + Runnable async = WhiteboardUtils.getService(wb, Runnable.class, new Predicate() { + @Override + public boolean apply(@Nullable Runnable input) { + return input instanceof AsyncIndexUpdate; + } + }); + assertNotNull(async); + async.run(); + root.refresh(); + } + +}