Major Description
the DestinationMap is locked via a publish send to a new dest:
"ActiveMQ NIO Worker 9808" #48458 daemon prio=9 os_prio=0 tid=0x00007f2c4c00e000 nid=0x4849 waiting for monitor entry [0x00007f2c59a78000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.activemq.broker.region.PrefetchSubscription.add(PrefetchSubscription.java:585) - waiting to lock <0x00000006c2921d78> (a java.lang.Object) at org.apache.activemq.broker.region.DurableTopicSubscription.add(DurableTopicSubscription.java:113) at org.apache.activemq.broker.region.Topic.addSubscription(Topic.java:160) at org.apache.activemq.broker.region.DestinationFilter.addSubscription(DestinationFilter.java:57) at org.apache.activemq.security.AuthorizationDestinationFilter.addSubscription(AuthorizationDestinationFilter.java:61) at org.apache.activemq.broker.region.AbstractRegion.addSubscriptionsForDestination(AbstractRegion.java:169) at org.apache.activemq.broker.region.TopicRegion.addSubscriptionsForDestination(TopicRegion.java:245) at org.apache.activemq.broker.region.AbstractRegion.addDestination(AbstractRegion.java:144) at org.apache.activemq.broker.region.RegionBroker.addDestination(RegionBroker.java:348) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:172) at org.apache.activemq.advisory.AdvisoryBroker.addDestination(AdvisoryBroker.java:198) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:172) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:172) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:172) at org.apache.activemq.broker.BrokerFilter.addDestination(BrokerFilter.java:172) at org.apache.activemq.security.AuthorizationBroker.addDestination(AuthorizationBroker.java:118) at org.apache.activemq.broker.MutableBrokerFilter.addDestination(MutableBrokerFilter.java:177) at org.apache.activemq.broker.util.LoggingBrokerPlugin.addDestination(LoggingBrokerPlugin.java:329) at org.apache.activemq.broker.MutableBrokerFilter.addDestination(MutableBrokerFilter.java:177) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:462) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:297) at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:152) at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96) at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:293) at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:152) at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:152) at org.apache.activemq.security.AuthorizationBroker.send(AuthorizationBroker.java:223) at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:157) at org.apache.activemq.broker.util.LoggingBrokerPlugin.send(LoggingBrokerPlugin.java:275) at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:157) at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:542) at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:768) at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:334) at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:188)
which is blocked on a durable sub: that is slow and is firing an advisory to that effect with the pending lock held.
The other thread has the destination lock and wants the pending lock to tell the sub about the new destination.
"ActiveMQ BrokerService[..] Task-6065" #48428 daemon prio=5 os_prio=0 tid=0x00007f2c640c5000 nid=0x4803 waiting on condition [0x00007f2c5898f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00000006c29630f0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireShared(AbstractQueuedSynchronizer.java:967) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1283) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at org.apache.activemq.broker.region.AbstractRegion.lookup(AbstractRegion.java:455) at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:415) at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:468) at org.apache.activemq.broker.jmx.ManagedRegionBroker.send(ManagedRegionBroker.java:297) at org.apache.activemq.advisory.AdvisoryBroker.fireAdvisory(AdvisoryBroker.java:634) at org.apache.activemq.advisory.AdvisoryBroker.slowConsumer(AdvisoryBroker.java:438) at org.apache.activemq.broker.BrokerFilter.slowConsumer(BrokerFilter.java:359) at org.apache.activemq.broker.BrokerFilter.slowConsumer(BrokerFilter.java:359) at org.apache.activemq.broker.BrokerFilter.slowConsumer(BrokerFilter.java:359) at org.apache.activemq.broker.BrokerFilter.slowConsumer(BrokerFilter.java:359) at org.apache.activemq.broker.MutableBrokerFilter.slowConsumer(MutableBrokerFilter.java:370) at org.apache.activemq.broker.util.LoggingBrokerPlugin.slowConsumer(LoggingBrokerPlugin.java:571) at org.apache.activemq.broker.MutableBrokerFilter.slowConsumer(MutableBrokerFilter.java:370) at org.apache.activemq.broker.region.BaseDestination.slowConsumer(BaseDestination.java:545) at org.apache.activemq.broker.region.PrefetchSubscription.dispatchPending(PrefetchSubscription.java:674) - locked <0x00000006c2921d78> (a java.lang.Object) at org.apache.activemq.broker.region.DurableTopicSubscription.dispatchPending(DurableTopicSubscription.java:282) at org.apache.activemq.broker.region.PrefetchSubscription.onDispatch(PrefetchSubscription.java:755) at org.apache.activemq.broker.region.PrefetchSubscription$3.onSuccess(PrefetchSubscription.java:717) at org.apache.activemq.broker.TransportConnection.processDispatch(TransportConnection.java:959) at org.apache.activemq.broker.TransportConnection.iterate(TransportConnection.java:988) at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:133) at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:48) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745)
All of the blocked threads result in disconnects/reconnects and exhaustion of available sockets.