diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/GroupImpl.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/GroupImpl.java index d61cfe124e..6e5a46373c 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/GroupImpl.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/GroupImpl.java @@ -48,6 +48,14 @@ class GroupImpl extends AuthorizableImpl implements Group { private static final Logger log = LoggerFactory.getLogger(GroupImpl.class); + static boolean USE_NEW = Boolean.getBoolean("GroupImpl.useNew"); + static { + if (USE_NEW) { + log.info("'GroupImpl.useNew' flag enabled."); + System.err.println("'GroupImpl.useNew' flag enabled."); + } + } + GroupImpl(String id, Tree tree, UserManagerImpl userManager) throws RepositoryException { super(id, tree, userManager); } diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipProvider.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipProvider.java index f2ab9292cd..954ec82338 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipProvider.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipProvider.java @@ -16,7 +16,9 @@ */ package org.apache.jackrabbit.oak.security.user; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.HashSet; import java.util.Iterator; import java.util.List; @@ -107,7 +109,7 @@ class MembershipProvider extends AuthorizableBaseProvider { private static final Logger log = LoggerFactory.getLogger(MembershipProvider.class); - private final MembershipWriter writer = new MembershipWriter(); + private final MembershipWriter writer = new MembershipWriter(GroupImpl.USE_NEW); /** * Creates a new membership provider @@ -387,14 +389,25 @@ class MembershipProvider extends AuthorizableBaseProvider { */ private abstract class MemberReferenceIterator extends AbstractLazyIterator { - private final Iterator trees; + private final Deque trees; + private Tree tree; private Iterator propertyValues; private MemberReferenceIterator(@Nonnull Tree groupTree) { - this.trees = Iterators.concat( - Iterators.singletonIterator(groupTree), - groupTree.getChild(REP_MEMBERS_LIST).getChildren().iterator() - ); + this.trees = new ArrayDeque<>(); + trees.push(groupTree); + } + + private Tree getTree() { + if (tree == null) { + tree = trees.poll(); + if (tree != null) { + for (Tree c : tree.getChildren()) { + trees.push(c); + } + } + } + return tree; } @Override @@ -403,17 +416,21 @@ class MembershipProvider extends AuthorizableBaseProvider { while (next == null) { if (propertyValues == null) { // check if there are more trees that can provide a rep:members property - if (!trees.hasNext()) { + Tree t = getTree(); + if (t == null) { // if not, we're done break; } - PropertyState property = trees.next().getProperty(REP_MEMBERS); + PropertyState property = t.getProperty(REP_MEMBERS); if (property != null) { propertyValues = property.getValue(Type.STRINGS).iterator(); + } else { + tree = null; } } else if (!propertyValues.hasNext()) { // if there are no more values left, reset the iterator propertyValues = null; + tree = null; } else { String value = propertyValues.next(); if (hasProcessedReference(value)) { diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipWriter.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipWriter.java index 05a2888bd3..ec5f53566d 100644 --- a/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipWriter.java +++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/security/user/MembershipWriter.java @@ -16,26 +16,31 @@ */ package org.apache.jackrabbit.oak.security.user; +import static org.apache.jackrabbit.oak.api.Type.NAME; +import static com.google.common.collect.Lists.newLinkedList; + +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import javax.annotation.Nonnull; import javax.jcr.RepositoryException; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import org.apache.jackrabbit.JcrConstants; import org.apache.jackrabbit.oak.api.PropertyState; import org.apache.jackrabbit.oak.api.Tree; import org.apache.jackrabbit.oak.api.Type; -import org.apache.jackrabbit.oak.spi.security.user.UserConstants; import org.apache.jackrabbit.oak.plugins.memory.PropertyBuilder; +import org.apache.jackrabbit.oak.plugins.tree.TreeUtil; +import org.apache.jackrabbit.oak.spi.security.user.UserConstants; import com.google.common.collect.Iterators; - -import static org.apache.jackrabbit.oak.api.Type.NAME; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; /** * @see MembershipProvider to more details. @@ -44,13 +49,21 @@ public class MembershipWriter { public static final int DEFAULT_MEMBERSHIP_THRESHOLD = 100; + private final WriterStrategy writer; + + public MembershipWriter(boolean useTreeWriter) { + if (useTreeWriter) { + writer = new TreeWriter(); + } else { + writer = new ListWriter(); + } + } + /** - * size of the membership threshold after which a new overflow node is created. + * Sets the size of the membership threshold after which a new overflow node is created. */ - private int membershipSizeThreshold = DEFAULT_MEMBERSHIP_THRESHOLD; - public void setMembershipSizeThreshold(int membershipSizeThreshold) { - this.membershipSizeThreshold = membershipSizeThreshold; + writer.setMembershipSizeThreshold(membershipSizeThreshold); } /** @@ -76,110 +89,7 @@ public class MembershipWriter { * @throws RepositoryException if an error occurs */ Set addMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) throws RepositoryException { - // check all possible rep:members properties for the new member and also find the one with the least values - Tree membersList = groupTree.getChild(UserConstants.REP_MEMBERS_LIST); - Iterator trees = Iterators.concat( - Iterators.singletonIterator(groupTree), - membersList.getChildren().iterator() - ); - - Set failed = new HashSet(memberIds.size()); - int bestCount = membershipSizeThreshold; - PropertyState bestProperty = null; - Tree bestTree = null; - - // remove existing memberIds from the map and find best-matching tree - // for the insertion of the new members. - while (trees.hasNext() && !memberIds.isEmpty()) { - Tree t = trees.next(); - PropertyState refs = t.getProperty(UserConstants.REP_MEMBERS); - if (refs != null) { - int numRefs = 0; - for (String ref : refs.getValue(Type.WEAKREFERENCES)) { - String id = memberIds.remove(ref); - if (id != null) { - failed.add(id); - if (memberIds.isEmpty()) { - break; - } - } - numRefs++; - } - if (numRefs < bestCount) { - bestCount = numRefs; - bestProperty = refs; - bestTree = t; - } - } - } - - // update member content structure by starting inserting new member IDs - // with the best-matching property and create new member-ref-nodes as needed. - if (!memberIds.isEmpty()) { - PropertyBuilder propertyBuilder; - int propCnt; - if (bestProperty == null) { - // we don't have a good candidate to store the new members. - // so there are no members at all or all are full - if (!groupTree.hasProperty(UserConstants.REP_MEMBERS)) { - bestTree = groupTree; - } else { - bestTree = createMemberRefTree(groupTree, membersList); - } - propertyBuilder = PropertyBuilder.array(Type.WEAKREFERENCE, UserConstants.REP_MEMBERS); - propCnt = 0; - } else { - propertyBuilder = PropertyBuilder.copy(Type.WEAKREFERENCE, bestProperty); - propCnt = bestCount; - } - // if adding all new members to best-property would exceed the threshold - // the new ids need to be distributed to different member-ref-nodes - // for simplicity this is achieved by introducing new tree(s) - if ((propCnt + memberIds.size()) > membershipSizeThreshold) { - while (!memberIds.isEmpty()) { - Set s = new HashSet(); - Iterator it = memberIds.keySet().iterator(); - while (propCnt < membershipSizeThreshold && it.hasNext()) { - s.add(it.next()); - it.remove(); - propCnt++; - } - propertyBuilder.addValues(s); - bestTree.setProperty(propertyBuilder.getPropertyState()); - - if (it.hasNext()) { - // continue filling the next (new) node + propertyBuilder pair - propCnt = 0; - bestTree = createMemberRefTree(groupTree, membersList); - propertyBuilder = PropertyBuilder.array(Type.WEAKREFERENCE, UserConstants.REP_MEMBERS); - } - } - } else { - propertyBuilder.addValues(memberIds.keySet()); - bestTree.setProperty(propertyBuilder.getPropertyState()); - } - } - return failed; - } - - private static Tree createMemberRefTree(@Nonnull Tree groupTree, @Nonnull Tree membersList) { - if (!membersList.exists()) { - membersList = groupTree.addChild(UserConstants.REP_MEMBERS_LIST); - membersList.setProperty(JcrConstants.JCR_PRIMARYTYPE, UserConstants.NT_REP_MEMBER_REFERENCES_LIST, NAME); - } - Tree refTree = membersList.addChild(nextRefNodeName(membersList)); - refTree.setProperty(JcrConstants.JCR_PRIMARYTYPE, UserConstants.NT_REP_MEMBER_REFERENCES, NAME); - return refTree; - } - - private static String nextRefNodeName(@Nonnull Tree membersList) { - // keep node names linear - int i = 0; - String name = String.valueOf(i); - while (membersList.hasChild(name)) { - name = String.valueOf(++i); - } - return name; + return writer.addMembers(groupTree, memberIds); } /** @@ -203,35 +113,367 @@ public class MembershipWriter { * @return the set of member IDs that was not successfully processed. */ Set removeMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) { - Tree membersList = groupTree.getChild(UserConstants.REP_MEMBERS_LIST); - Iterator trees = Iterators.concat( - Iterators.singletonIterator(groupTree), - membersList.getChildren().iterator() - ); - while (trees.hasNext() && !memberIds.isEmpty()) { - Tree t = trees.next(); + return writer.removeMembers(groupTree, memberIds); + } + + private static interface WriterStrategy { + + void setMembershipSizeThreshold(int membershipSizeThreshold); + + Set addMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) + throws RepositoryException; + + Set removeMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds); + + } + + static class TreeWriter implements WriterStrategy { + + private int membershipSizeThreshold = DEFAULT_MEMBERSHIP_THRESHOLD; + + // a132cbbd-6a2c-3981-965a-239e22fba6c7 + private static int MAX_LEVEL = 35; + + public void setMembershipSizeThreshold(int membershipSizeThreshold) { + this.membershipSizeThreshold = membershipSizeThreshold; + } + + @Override + public Set addMembers(@Nonnull Tree groupTree, final @Nonnull Map memberIds) + throws RepositoryException { + Set failed = new HashSet(memberIds.size()); + addMembers(groupTree, -1, membershipSizeThreshold, MAX_LEVEL, memberIds, failed, memberIds.keySet(), false); + return failed; + } + + static void addMembers(@Nonnull Tree t, int level, int threshold, int maxLevel, + @Nonnull Map memberIds, @Nonnull Set failed, @Nonnull Collection values, + boolean force) { + + // if property exists, we're sure this is a leaf + PropertyState refs = t.getProperty(UserConstants.REP_MEMBERS); + if (force || refs != null) { + List newVals = filterAndGet(refs, values, memberIds, failed); + if (newVals == null) { + return; + } + newVals.addAll(values); + + if (newVals.size() <= threshold || level >= maxLevel) { + setRepMembers(t, newVals); + + } else { + // merge & split + t.removeProperty(UserConstants.REP_MEMBERS); + if (level == -1) { + t = t.addChild(UserConstants.REP_MEMBERS_LIST); + } + // change node type from 'refs' to 'ref list' + t.setProperty(JcrConstants.JCR_PRIMARYTYPE, UserConstants.NT_REP_MEMBER_REFERENCES_LIST, NAME); + addMembersAsTree(t, level, threshold, maxLevel, memberIds, failed, newVals, true); + } + } else if (isLeafType(t, level)) { + addMembers(t, level, threshold, maxLevel, memberIds, failed, values, true); + } else { + // continue going down the tree + if (level == -1) { + t = getOrAdd(t, UserConstants.REP_MEMBERS_LIST, UserConstants.NT_REP_MEMBER_REFERENCES_LIST, false); + } + addMembersAsTree(t, level, threshold, maxLevel, memberIds, failed, values, false); + } + } + + private static void addMembersAsTree(Tree t, int level, int threshold, int maxLevel, + Map memberIds, Set failed, Collection values, boolean forceAdd) { + level++; + Map> groupped = groupByKey(values, level); + for (Entry> e : groupped.entrySet()) { + String key = e.getKey(); + Tree c = getOrAdd(t, key, UserConstants.NT_REP_MEMBER_REFERENCES, forceAdd); + addMembers(c, level, threshold, maxLevel, memberIds, failed, e.getValue(), true); + } + } + + private static List filterAndGet(PropertyState refs, Collection values, + Map memberIds, Set failed) { + if (refs == null) { + return newLinkedList(); + } + for (String ref : refs.getValue(Type.WEAKREFERENCES)) { + if (values.contains(ref)) { + failed.add(memberIds.get(ref)); + values.remove(ref); + // check if I can stop the iteration early + if (values.isEmpty()) { + return null; + } + } + } + return newLinkedList(refs.getValue(Type.WEAKREFERENCES)); + } + + private static Tree getOrAdd(Tree t, String name, String type, boolean force) { + Tree c; + if (!force && t.hasChild(name)) { + c = t.getChild(name); + } else { + c = t.addChild(name); + c.setProperty(JcrConstants.JCR_PRIMARYTYPE, type, NAME); + } + return c; + } + + private static void setRepMembers(Tree t, Iterable ids) { + PropertyBuilder propertyBuilder; + propertyBuilder = PropertyBuilder.array(Type.WEAKREFERENCE, UserConstants.REP_MEMBERS); + propertyBuilder.addValues(ids); + t.setProperty(propertyBuilder.getPropertyState()); + } + + private static String idToKey(String id, int level) { + return id.charAt(level) + ""; + } + + private static boolean isLeafType(Tree t, int level) { + if (level == -1) { + return !t.hasChild(UserConstants.REP_MEMBERS_LIST); + } else { + String pt = TreeUtil.getName(t, JcrConstants.JCR_PRIMARYTYPE); + return UserConstants.NT_REP_MEMBER_REFERENCES.equals(pt); + } + } + + private static Map> groupByKey(Collection uuids, int level) { + Map> ret = Maps.newHashMapWithExpectedSize(16); + for (String uuid : uuids) { + String key = idToKey(uuid, level); + Collection vals = ret.get(key); + if (vals == null) { + vals = newLinkedList(); + } + vals.add(uuid); + ret.put(key, vals); + } + return ret; + } + + @Override + public Set removeMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) { + Set failed = Sets.newHashSet(); + Set rms = Sets.newHashSet(memberIds.keySet()); + removeMembers(groupTree, -1, memberIds, failed, rms); + return failed; + } + + private static void removeMembers(Tree t, int level, Map memberIds, Set failed, Collection rms) { + // if property exists, we're sure this is a leaf PropertyState refs = t.getProperty(UserConstants.REP_MEMBERS); if (refs != null) { - PropertyBuilder prop = PropertyBuilder.copy(Type.WEAKREFERENCE, refs); - Iterator> it = memberIds.entrySet().iterator(); - while (it.hasNext() && !prop.isEmpty()) { - String memberContentId = it.next().getKey(); - if (prop.hasValue(memberContentId)) { - prop.removeValue(memberContentId); + Set vals = Sets.newHashSet(refs.getValue(Type.WEAKREFERENCES)); + boolean dirty = false; + Iterator it = vals.iterator(); + while (it.hasNext()) { + String v = it.next(); + if (rms.remove(v)) { it.remove(); + dirty = true; + if (rms.isEmpty()) { + break; + } + } + } + for (String k : rms) { + failed.add(memberIds.get(k)); + } + if (dirty) { + if (vals.isEmpty()) { + if (level < 0) { + // TODO remove OR set to empty? + t.removeProperty(UserConstants.REP_MEMBERS); + } else { + // TODO more aggressive pruning in case of deletes + t.remove(); + } + } else { + setRepMembers(t, vals); + } + } + } else if (isLeafType(t, level)) { + for (String k : rms) { + failed.add(memberIds.get(k)); + } + + } else { + // continue going down the tree + if (level == -1) { + t = t.getChild(UserConstants.REP_MEMBERS_LIST); + if (!t.exists()) { + return; + } + } + level++; + Map> groupped = groupByKey(rms, level); + for (Entry> e : groupped.entrySet()) { + String key = e.getKey(); + if (t.hasChild(key)) { + removeMembers(t.getChild(key), level, memberIds, failed, e.getValue()); + } + } + } + } + } + + private static class ListWriter implements WriterStrategy { + + private int membershipSizeThreshold = DEFAULT_MEMBERSHIP_THRESHOLD; + + public void setMembershipSizeThreshold(int membershipSizeThreshold) { + this.membershipSizeThreshold = membershipSizeThreshold; + } + + @Override + public Set addMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) + throws RepositoryException { + + // check all possible rep:members properties for the new member and also find the one with the least values + Tree membersList = groupTree.getChild(UserConstants.REP_MEMBERS_LIST); + Iterator trees = Iterators.concat( + Iterators.singletonIterator(groupTree), + membersList.getChildren().iterator() + ); + + Set failed = new HashSet(memberIds.size()); + int bestCount = membershipSizeThreshold; + PropertyState bestProperty = null; + Tree bestTree = null; + + // remove existing memberIds from the map and find best-matching tree + // for the insertion of the new members. + while (trees.hasNext() && !memberIds.isEmpty()) { + Tree t = trees.next(); + PropertyState refs = t.getProperty(UserConstants.REP_MEMBERS); + if (refs != null) { + int numRefs = 0; + for (String ref : refs.getValue(Type.WEAKREFERENCES)) { + String id = memberIds.remove(ref); + if (id != null) { + failed.add(id); + if (memberIds.isEmpty()) { + break; + } + } + numRefs++; + } + if (numRefs < bestCount) { + bestCount = numRefs; + bestProperty = refs; + bestTree = t; } } - if (prop.isEmpty()) { - if (t == groupTree) { - t.removeProperty(UserConstants.REP_MEMBERS); + } + + // update member content structure by starting inserting new member IDs + // with the best-matching property and create new member-ref-nodes as needed. + if (!memberIds.isEmpty()) { + PropertyBuilder propertyBuilder; + int propCnt; + if (bestProperty == null) { + // we don't have a good candidate to store the new members. + // so there are no members at all or all are full + if (!groupTree.hasProperty(UserConstants.REP_MEMBERS)) { + bestTree = groupTree; } else { - t.remove(); + bestTree = createMemberRefTree(groupTree, membersList); } + propertyBuilder = PropertyBuilder.array(Type.WEAKREFERENCE, UserConstants.REP_MEMBERS); + propCnt = 0; } else { - t.setProperty(prop.getPropertyState()); + propertyBuilder = PropertyBuilder.copy(Type.WEAKREFERENCE, bestProperty); + propCnt = bestCount; } + // if adding all new members to best-property would exceed the threshold + // the new ids need to be distributed to different member-ref-nodes + // for simplicity this is achieved by introducing new tree(s) + if ((propCnt + memberIds.size()) > membershipSizeThreshold) { + while (!memberIds.isEmpty()) { + Set s = new HashSet(); + Iterator it = memberIds.keySet().iterator(); + while (propCnt < membershipSizeThreshold && it.hasNext()) { + s.add(it.next()); + it.remove(); + propCnt++; + } + propertyBuilder.addValues(s); + bestTree.setProperty(propertyBuilder.getPropertyState()); + + if (it.hasNext()) { + // continue filling the next (new) node + propertyBuilder pair + propCnt = 0; + bestTree = createMemberRefTree(groupTree, membersList); + propertyBuilder = PropertyBuilder.array(Type.WEAKREFERENCE, UserConstants.REP_MEMBERS); + } + } + } else { + propertyBuilder.addValues(memberIds.keySet()); + bestTree.setProperty(propertyBuilder.getPropertyState()); + } + } + return failed; + } + + private static Tree createMemberRefTree(@Nonnull Tree groupTree, @Nonnull Tree membersList) { + if (!membersList.exists()) { + membersList = groupTree.addChild(UserConstants.REP_MEMBERS_LIST); + membersList.setProperty(JcrConstants.JCR_PRIMARYTYPE, UserConstants.NT_REP_MEMBER_REFERENCES_LIST, NAME); } + Tree refTree = membersList.addChild(nextRefNodeName(membersList)); + refTree.setProperty(JcrConstants.JCR_PRIMARYTYPE, UserConstants.NT_REP_MEMBER_REFERENCES, NAME); + return refTree; } - return Sets.newHashSet(memberIds.values()); + + private static String nextRefNodeName(@Nonnull Tree membersList) { + // keep node names linear + int i = 0; + String name = String.valueOf(i); + while (membersList.hasChild(name)) { + name = String.valueOf(++i); + } + return name; + } + + @Override + public Set removeMembers(@Nonnull Tree groupTree, @Nonnull Map memberIds) { + Tree membersList = groupTree.getChild(UserConstants.REP_MEMBERS_LIST); + Iterator trees = Iterators.concat( + Iterators.singletonIterator(groupTree), + membersList.getChildren().iterator() + ); + while (trees.hasNext() && !memberIds.isEmpty()) { + Tree t = trees.next(); + PropertyState refs = t.getProperty(UserConstants.REP_MEMBERS); + if (refs != null) { + PropertyBuilder prop = PropertyBuilder.copy(Type.WEAKREFERENCE, refs); + Iterator> it = memberIds.entrySet().iterator(); + while (it.hasNext() && !prop.isEmpty()) { + String memberContentId = it.next().getKey(); + if (prop.hasValue(memberContentId)) { + prop.removeValue(memberContentId); + it.remove(); + } + } + if (prop.isEmpty()) { + if (t == groupTree) { + t.removeProperty(UserConstants.REP_MEMBERS); + } else { + t.remove(); + } + } else { + t.setProperty(prop.getPropertyState()); + } + } + } + return Sets.newHashSet(memberIds.values()); + } + } } \ No newline at end of file diff --git a/oak-core/src/main/resources/org/apache/jackrabbit/oak/builtin_nodetypes.cnd b/oak-core/src/main/resources/org/apache/jackrabbit/oak/builtin_nodetypes.cnd index 0214cbc788..6771a502de 100644 --- a/oak-core/src/main/resources/org/apache/jackrabbit/oak/builtin_nodetypes.cnd +++ b/oak-core/src/main/resources/org/apache/jackrabbit/oak/builtin_nodetypes.cnd @@ -781,6 +781,7 @@ */ [rep:MemberReferencesList] + * (rep:MemberReferences) = rep:MemberReferences protected COPY + + * (rep:MemberReferencesList) = rep:MemberReferences protected COPY /* @since oak 1.7.0 */ /** * @since oak 1.4 diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/GroupImplTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/GroupImplTest.java index 0dead27cab..18f28e3643 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/GroupImplTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/GroupImplTest.java @@ -16,11 +16,19 @@ */ package org.apache.jackrabbit.oak.security.user; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + import java.security.Principal; +import java.util.Arrays; import java.util.Iterator; +import java.util.List; +import java.util.Set; import java.util.UUID; -import com.google.common.collect.Iterators; import org.apache.jackrabbit.api.security.user.Authorizable; import org.apache.jackrabbit.api.security.user.Group; import org.apache.jackrabbit.oak.AbstractSecurityTest; @@ -28,10 +36,8 @@ import org.apache.jackrabbit.oak.spi.security.principal.EveryonePrincipal; import org.junit.Test; import org.mockito.Mockito; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertTrue; +import com.google.common.collect.Iterators; +import com.google.common.collect.Lists; public class GroupImplTest extends AbstractSecurityTest { @@ -121,4 +127,49 @@ public class GroupImplTest extends AbstractSecurityTest { Iterator members = groupPrincipal.getMembers(); assertTrue(Iterators.elementsEqual(group.getMembers(), members)); } + + @Test + public void testMembersSetGetRmInline() throws Exception { + addMembers(50); + } + + @Test + public void testMembersSetGetRmTree1() throws Exception { + addMembers(150); + } + + @Test + public void testMembersSetGetRmTree2() throws Exception { + addMembers(1510); + } + + private void addMembers(int size) throws Exception { + String[] tests = new String[size]; + for (int i = 0; i < size; i++) { + String id = "user" + System.currentTimeMillis() + "-" + i; + tests[i] = id; + assertTrue(uMgr.createGroup(id) != null); + } + Arrays.sort(tests); + + Set res1 = group.addMembers(tests); + assertTrue("unable to add [" + res1.size() + "] " + res1, res1.isEmpty()); + + List out = Lists.newArrayList(group.getMembers()); + assertEquals(size, out.size()); + + String[] got = new String[size]; + int i = 0; + for (Authorizable a : out) { + got[i++] = a.getID(); + } + Arrays.sort(got); + assertArrayEquals("membership sets not equal", tests, got); + + Set res2 = group.removeMembers(tests); + assertTrue("unable to remove " + res2, res2.isEmpty()); + + List out2 = Lists.newArrayList(group.getMembers()); + assertEquals(0, out2.size()); + } } \ No newline at end of file diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTest.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTest.java index bc50ecc64d..cf50b6f78f 100644 --- a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTest.java +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTest.java @@ -56,7 +56,7 @@ public class MembershipWriterTest extends MembershipBaseTest { @Before public void before() throws Exception { super.before(); - writer = new MembershipWriter(); + writer = new MembershipWriter(false); // set the threshold low for testing writer.setMembershipSizeThreshold(SIZE_TH); } diff --git a/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTestIT.java b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTestIT.java new file mode 100644 index 0000000000..e4938984f2 --- /dev/null +++ b/oak-core/src/test/java/org/apache/jackrabbit/oak/security/user/MembershipWriterTestIT.java @@ -0,0 +1,188 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.jackrabbit.oak.security.user; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; + +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.jackrabbit.api.security.user.Group; +import org.apache.jackrabbit.oak.api.Tree; +import org.junit.Test; + +import com.google.common.collect.Maps; + +public class MembershipWriterTestIT extends MembershipBaseTest { + + @Test + public void testBenchAddMembersUnique() throws Exception { + assumeTrue(Boolean.getBoolean("MembershipWriterTestIT.testBenchAddMembersUnique")); + + Group grp = createGroup(); + + // [TREE] + // [ADD] #0 1000 times x 5 items. duration 36 ms. + // [ADD] #1 1000 times x 5 items. duration 24 ms. + // [ADD] #2 1000 times x 5 items. duration 17 ms. + // [ADD] #3 1000 times x 5 items. duration 21 ms. + // [ADD] #4 1000 times x 5 items. duration 15 ms. + // [ADD] #5 1000 times x 5 items. duration 23 ms. + // [ADD] #6 1000 times x 5 items. duration 13 ms. + // [ADD] #7 1000 times x 5 items. duration 17 ms. + // [ADD] #8 1000 times x 5 items. duration 17 ms. + // [ADD] #9 1000 times x 5 items. duration 39 ms. + // [ADD] #10 1000 times x 5 items. duration 46 ms. + // [ADD] #11 1000 times x 5 items. duration 16 ms. + // [ADD] #12 1000 times x 5 items. duration 16 ms. + // [ADD] #13 1000 times x 5 items. duration 15 ms. + // [ADD] #14 1000 times x 5 items. duration 21 ms. + // [ADD] #15 1000 times x 5 items. duration 34 ms. + // [ADD] #16 1000 times x 5 items. duration 13 ms. + // [ADD] #17 1000 times x 5 items. duration 16 ms. + // [ADD] #18 1000 times x 5 items. duration 19 ms. + // [ADD] #19 1000 times x 5 items. duration 23 ms. + // [ADD] #20 1000 times x 5 items. duration 20 ms. + // [ADD] #21 1000 times x 5 items. duration 36 ms. + // [ADD] #22 1000 times x 5 items. duration 33 ms. + // [ADD] #23 1000 times x 5 items. duration 25 ms. + // [ADD] #24 1000 times x 5 items. duration 29 ms. + + // [LIST] + // [ADD] #0 1000 times x 5 items. duration 120 ms. + // [ADD] #1 1000 times x 5 items. duration 200 ms. + // [ADD] #2 1000 times x 5 items. duration 248 ms. + // [ADD] #3 1000 times x 5 items. duration 347 ms. + // [ADD] #4 1000 times x 5 items. duration 415 ms. + // [ADD] #5 1000 times x 5 items. duration 476 ms. + // [ADD] #6 1000 times x 5 items. duration 659 ms. + // [ADD] #7 1000 times x 5 items. duration 736 ms. + // [ADD] #8 1000 times x 5 items. duration 790 ms. + // [ADD] #9 1000 times x 5 items. duration 860 ms. + // [ADD] #10 1000 times x 5 items. duration 946 ms. + // [ADD] #11 1000 times x 5 items. duration 1053 ms. + // [ADD] #12 1000 times x 5 items. duration 1154 ms. + // [ADD] #13 1000 times x 5 items. duration 1269 ms. + // [ADD] #14 1000 times x 5 items. duration 1624 ms. + // [ADD] #15 1000 times x 5 items. duration 1692 ms. + // [ADD] #16 1000 times x 5 items. duration 1633 ms. + // [ADD] #17 1000 times x 5 items. duration 1754 ms. + // [ADD] #18 1000 times x 5 items. duration 1888 ms. + // [ADD] #19 1000 times x 5 items. duration 2201 ms. + // [ADD] #20 1000 times x 5 items. duration 2369 ms. + // [ADD] #21 1000 times x 5 items. duration 2442 ms. + // [ADD] #22 1000 times x 5 items. duration 2652 ms. + // [ADD] #23 1000 times x 5 items. duration 2820 ms. + // [ADD] #24 1000 times x 5 items. duration 2983 ms. + + int times = 1000; + int size = 5; + boolean useTreeWriter = true; + + MembershipWriter writer = new MembershipWriter(useTreeWriter); + Tree t = getTree(grp); + + for (int op = 0; op < 25; op++) { + long total = 0; + for (int c = 0; c < times; c++) { + Map idMap = Maps.newHashMap(); + for (int i = 0; i < size; i++) { + String memberId = "user" + System.currentTimeMillis() + "-" + c + "-" + i; + idMap.put(getContentID(memberId), memberId); + } + long start = System.currentTimeMillis(); + Set res = writer.addMembers(t, idMap); + long dur = System.currentTimeMillis() - start; + total += dur; + assertTrue("unable to add " + res, res.isEmpty()); + } + System.err + .println("[ADD] #" + op + " " + times + " times x " + size + " items. duration " + total + " ms."); + } + } + + @Test + public void testBenchAddMembersExisting() throws Exception { + assumeTrue(Boolean.getBoolean("MembershipWriterTestIT.testBenchAddMembersExisting")); + + Group grp = createGroup(); + + // [TREE] + // [ADD] 10000 times x 5 samples | 50 items. duration 11ms (inlined) + // [ADD] 10000 times x 5 samples | 150 items. duration 24ms + // [ADD] 10000 times x 10 samples | 150 items. duration 42ms + // [ADD] 10000 times x 50 samples | 150 items. duration 135ms + // [ADD] 10000 times x 150 samples | 150 items. duration 250ms + + // [LIST] + // [ADD] 10000 times x 5 samples | 50 items. duration 10ms (inlined) + // [ADD] 10000 times x 5 samples | 150 items. duration 23ms + // [ADD] 10000 times x 10 samples | 150 items. duration 30ms + // [ADD] 10000 times x 50 samples | 150 items. duration 65ms + // [ADD] 10000 times x 150 samples | 150 items. duration 140ms + + int batch = 150; + int size = 150; + boolean useTreeWriter = true; + int times = 10000; + + MembershipWriter writer = new MembershipWriter(useTreeWriter); + + // Setup adds all members beforehand + Map ids = Maps.newHashMap(); + String[] keys = new String[size]; + for (int i = 0; i < size; i++) { + String memberId = "user" + System.currentTimeMillis() + "-" + i; + String key = getContentID(memberId); + ids.put(key, memberId); + keys[i] = key; + } + assertTrue("unable to setup ", writer.addMembers(getTree(grp), ids).isEmpty()); + root.commit(); + Tree t = getTree(grp); + + long tg = 0; + for (int cg = 1; cg <= 1000; cg++) { + long total = 0; + Random r = new Random(); + for (int c = 0; c < times; c++) { + long start = System.currentTimeMillis(); + + Map sample = Maps.newHashMap(); + for (int i = 0; i < batch; i++) { + String key = keys[r.nextInt(size)]; + sample.put(key, "" + i); + } + int samples = sample.size(); + + Set res = writer.addMembers(t, sample); + + long dur = System.currentTimeMillis() - start; + total += dur; + assertTrue("should not add any (" + res.size() + " vs " + samples + ")! " + res, res.size() == samples); + } + + tg += total; + long avg = tg / cg; + System.err.println("[AddExisting] " + times + " times x " + batch + " samples | " + size + + " items. duration " + total + " ms. -- #" + cg + ": avg " + avg + " ms."); + } + } + +} \ No newline at end of file