diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMemberTest.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMemberTest.java
index 9dd7fa1c17..736b49038a 100644
--- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMemberTest.java
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMemberTest.java
@@ -56,7 +56,7 @@ public class AddMemberTest extends AddMembersTest {
     }
 
     @Override
-    protected void addMembers(@Nonnull UserManager userManager, @Nonnull Group group, @Nonnull Session s) throws Exception {
+    protected void addMembers(@Nonnull UserManager userManager, @Nonnull Group group, @Nonnull Session s, long index) throws Exception {
         int j = 1;
         for (int i = 0; i <= numberOfMembers; i++) {
             String userPath = userPaths.get(random.nextInt(numberOfMembers));
diff --git a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMembersTest.java b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMembersTest.java
index a110de778a..c074ea9522 100644
--- a/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMembersTest.java
+++ b/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/AddMembersTest.java
@@ -19,6 +19,9 @@ package org.apache.jackrabbit.oak.benchmark;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicLong;
+
 import javax.annotation.Nonnull;
 import javax.jcr.Node;
 import javax.jcr.Repository;
@@ -42,6 +45,7 @@ import org.apache.jackrabbit.oak.spi.security.user.UserConfiguration;
 import org.apache.jackrabbit.oak.spi.xml.ImportBehavior;
 import org.apache.jackrabbit.oak.spi.xml.ProtectedItemImporter;
 import org.apache.jackrabbit.util.Text;
+import org.junit.Assert;
 
 /**
  * Test the performance of adding a configured number of members to groups. The
@@ -62,13 +66,14 @@ public class AddMembersTest extends AbstractTest {
     static final String GROUP = "group";
     static final int GROUP_CNT = 100;
     static final int DEFAULT_BATCH_SIZE = 1;
+    private static AtomicLong index = new AtomicLong();
 
     final Random random = new Random();
     final int numberOfMembers;
     final int batchSize;
     private final String importBehavior;
 
-    private final List<String> groupPaths = new ArrayList(GROUP_CNT);
+    private final List<String> groupPaths = new ArrayList<>(GROUP_CNT);
 
     public AddMembersTest(int numberOfMembers, int batchSize, @Nonnull String importBehavior) {
         this.numberOfMembers = numberOfMembers;
@@ -153,7 +158,7 @@ public class AddMembersTest extends AbstractTest {
             UserManager userManager = ((JackrabbitSession) s).getUserManager();
             String groupPath = groupPaths.get(random.nextInt(GROUP_CNT));
             Group g = (Group) userManager.getAuthorizableByPath(groupPath);
-            addMembers(userManager, g, s);
+            addMembers(userManager, g, s, index.getAndIncrement());
         } catch (RepositoryException e) {
             if (s.hasPendingChanges()) {
                 s.refresh(false);
@@ -165,18 +170,20 @@ public class AddMembersTest extends AbstractTest {
         }
     }
 
-    protected void addMembers(@Nonnull UserManager userManger, @Nonnull Group group, @Nonnull Session s) throws Exception {
+    protected void addMembers(@Nonnull UserManager userManger, @Nonnull Group group, @Nonnull Session s, long index) throws Exception {
         for (int i = 0; i <= numberOfMembers; i++) {
+            Set<String> failed;
             if (batchSize <= DEFAULT_BATCH_SIZE) {
-                group.addMembers(USER + i);
+                failed = group.addMembers(USER + i+ "_" + index);
             } else {
                 List<String> ids = new ArrayList<String>(batchSize);
                 for (int j = 0; j < batchSize && i <= numberOfMembers; j++) {
-                    ids.add(USER + i);
+                    ids.add(USER + i+ "_" + index);
                     i++;
                 }
-                group.addMembers(ids.toArray(new String[ids.size()]));
+                failed = group.addMembers(ids.toArray(new String[ids.size()]));
             }
+            Assert.assertTrue("Group " + group.getID() + ": unable to add: " + failed, failed.isEmpty());
             s.save();
         }
     }