Found one Java-level deadlock: ============================= "aysnc-index-update-async": waiting for ownable synchronizer 0x00000007e12c3d28, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), which is held by "Thread-516" "Thread-516": waiting for ownable synchronizer 0x00000007e127c5f8, (a java.util.concurrent.locks.ReentrantLock$NonfairSync), which is held by "Thread-512" "Thread-512": waiting for ownable synchronizer 0x00000007e129af00, (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync), which is held by "aysnc-index-update-async" Java stack information for the threads listed above: =================================================== "aysnc-index-update-async": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007e12c3d28> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks$TreeLock.lock(TreeNodeDocumentLocks.java:180) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks.acquire(TreeNodeDocumentLocks.java:74) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks.acquire(TreeNodeDocumentLocks.java:39) at org.apache.jackrabbit.oak.plugins.document.cache.NodeDocumentCache.putIfNewer(NodeDocumentCache.java:159) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.queryInternal(MongoDocumentStore.java:624) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.query(MongoDocumentStore.java:540) at org.apache.jackrabbit.oak.plugins.document.util.LeaseCheckDocumentStoreWrapper.query(LeaseCheckDocumentStoreWrapper.java:85) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.diffManyChildren(DocumentNodeStore.java:2252) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.diffImpl(DocumentNodeStore.java:2218) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.access$400(DocumentNodeStore.java:123) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore$9.call(DocumentNodeStore.java:1435) at org.apache.jackrabbit.oak.plugins.document.MemoryDiffCache$1.call(MemoryDiffCache.java:81) at org.apache.jackrabbit.oak.plugins.document.MemoryDiffCache$1.call(MemoryDiffCache.java:75) at org.apache.jackrabbit.oak.cache.CacheLIRS$Segment.load(CacheLIRS.java:1004) at org.apache.jackrabbit.oak.cache.CacheLIRS$Segment.get(CacheLIRS.java:969) - locked <0x00000007e742e7f0> (a java.lang.Object) at org.apache.jackrabbit.oak.cache.CacheLIRS.get(CacheLIRS.java:280) at org.apache.jackrabbit.oak.plugins.document.persistentCache.NodeCache.get(NodeCache.java:149) at org.apache.jackrabbit.oak.plugins.document.MemoryDiffCache.getChanges(MemoryDiffCache.java:75) at org.apache.jackrabbit.oak.plugins.document.TieredDiffCache.getChanges(TieredDiffCache.java:50) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.compare(DocumentNodeStore.java:1430) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeState.compareAgainstBaseState(DocumentNodeState.java:344) at org.apache.jackrabbit.oak.spi.commit.EditorDiff.childNodeChanged(EditorDiff.java:148) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore$14.childNodeChanged(DocumentNodeStore.java:2127) at org.apache.jackrabbit.oak.plugins.document.DiffCache.parseJsopDiff(DiffCache.java:122) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.dispatch(DocumentNodeStore.java:2113) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.compare(DocumentNodeStore.java:1430) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeState.compareAgainstBaseState(DocumentNodeState.java:344) at org.apache.jackrabbit.oak.spi.commit.EditorDiff.process(EditorDiff.java:52) at org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.updateIndex(AsyncIndexUpdate.java:427) at org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate.run(AsyncIndexUpdate.java:357) - locked <0x00000007e1ffec58> (a org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) "Thread-516": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007e127c5f8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:867) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1197) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:214) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:290) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks$TreeLock.lock(TreeNodeDocumentLocks.java:180) at org.apache.jackrabbit.oak.plugins.document.locks.BulkLock.lock(BulkLock.java:40) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks.acquire(TreeNodeDocumentLocks.java:110) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.bulkUpdate(MongoDocumentStore.java:935) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.createOrUpdate(MongoDocumentStore.java:891) at org.apache.jackrabbit.oak.plugins.document.util.LeaseCheckDocumentStoreWrapper.createOrUpdate(LeaseCheckDocumentStoreWrapper.java:133) at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:368) at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:262) at org.apache.jackrabbit.oak.plugins.document.Commit.applyInternal(Commit.java:231) at org.apache.jackrabbit.oak.plugins.document.Commit.apply(Commit.java:220) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:292) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:262) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.access$300(DocumentNodeStoreBranch.java:57) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.merge(DocumentNodeStoreBranch.java:499) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge0(DocumentNodeStoreBranch.java:182) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge(DocumentNodeStoreBranch.java:118) at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.merge(DocumentRootBuilder.java:158) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.merge(DocumentNodeStore.java:1508) at org.apache.jackrabbit.oak.core.MutableRoot.commit(MutableRoot.java:247) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.commit(SessionDelegate.java:347) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.save(SessionDelegate.java:494) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.performVoid(SessionImpl.java:419) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.performVoid(SessionDelegate.java:274) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.save(SessionImpl.java:416) at org.apache.jackrabbit.oak.jcr.ConcurrentAddIT$Worker.run(ConcurrentAddIT.java:128) at java.lang.Thread.run(Thread.java:745) "Thread-512": at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000007e129af00> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:834) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:964) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:731) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks$TreeLock.lock(TreeNodeDocumentLocks.java:178) at org.apache.jackrabbit.oak.plugins.document.locks.BulkLock.lock(BulkLock.java:40) at org.apache.jackrabbit.oak.plugins.document.locks.TreeNodeDocumentLocks.acquire(TreeNodeDocumentLocks.java:110) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.bulkUpdate(MongoDocumentStore.java:935) at org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore.createOrUpdate(MongoDocumentStore.java:891) at org.apache.jackrabbit.oak.plugins.document.util.LeaseCheckDocumentStoreWrapper.createOrUpdate(LeaseCheckDocumentStoreWrapper.java:133) at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:368) at org.apache.jackrabbit.oak.plugins.document.Commit.applyToDocumentStore(Commit.java:262) at org.apache.jackrabbit.oak.plugins.document.Commit.applyInternal(Commit.java:231) at org.apache.jackrabbit.oak.plugins.document.Commit.apply(Commit.java:220) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:292) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.persist(DocumentNodeStoreBranch.java:262) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.access$300(DocumentNodeStoreBranch.java:57) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch$InMemory.merge(DocumentNodeStoreBranch.java:499) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge0(DocumentNodeStoreBranch.java:182) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBranch.merge(DocumentNodeStoreBranch.java:118) at org.apache.jackrabbit.oak.plugins.document.DocumentRootBuilder.merge(DocumentRootBuilder.java:158) at org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore.merge(DocumentNodeStore.java:1508) at org.apache.jackrabbit.oak.core.MutableRoot.commit(MutableRoot.java:247) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.commit(SessionDelegate.java:347) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.save(SessionDelegate.java:494) at org.apache.jackrabbit.oak.jcr.session.SessionImpl$8.performVoid(SessionImpl.java:419) at org.apache.jackrabbit.oak.jcr.delegate.SessionDelegate.performVoid(SessionDelegate.java:274) at org.apache.jackrabbit.oak.jcr.session.SessionImpl.save(SessionImpl.java:416) at org.apache.jackrabbit.oak.jcr.ConcurrentAddIT$Worker.run(ConcurrentAddIT.java:128) at java.lang.Thread.run(Thread.java:745) Found 1 deadlock.