Major Description
Stack traces of all threads when the test is hanging. Looks like the issue is with concurrent access to the message properties hashmap during message copy.
ActiveMQ Transport: tcp:///127.0.0.1:56744[1] where all main: [1] sun.misc.Unsafe.park (native method) [2] java.util.concurrent.locks.LockSupport.parkNanos (LockSupport.java:213) [3] java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos (AbstractQueuedSynchronizer.java:958) [4] java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos (AbstractQueuedSynchronizer.java:1,250) [5] java.util.concurrent.CountDownLatch.await (CountDownLatch.java:264) [6] org.apache.activemq.broker.TransportConnection.stop (TransportConnection.java:905) [7] org.apache.activemq.util.ServiceStopper.stop (ServiceStopper.java:41) [8] org.apache.activemq.broker.TransportConnector.stop (TransportConnector.java:281) [9] org.apache.activemq.util.ServiceStopper.stop (ServiceStopper.java:41) [10] org.apache.activemq.broker.BrokerService.stopAllConnectors (BrokerService.java:1,536) [11] org.apache.activemq.broker.BrokerService.stop (BrokerService.java:508) [12] org.apache.activemq.bugs.AMQ2021Test.tearDown (AMQ2021Test.java:94) [13] junit.framework.TestCase.runBare (TestCase.java:140) [14] junit.framework.TestResult$1.protect (TestResult.java:110) [15] junit.framework.TestResult.runProtected (TestResult.java:128) [16] junit.framework.TestResult.run (TestResult.java:113) [17] junit.framework.TestCase.run (TestCase.java:124) [18] junit.framework.TestSuite.runTest (TestSuite.java:232) [19] junit.framework.TestSuite.run (TestSuite.java:227) [20] org.junit.internal.runners.JUnit38ClassRunner.run (JUnit38ClassRunner.java:79) [21] org.apache.maven.surefire.junit4.JUnit4TestSet.execute (JUnit4TestSet.java:62) [22] org.apache.maven.surefire.suite.AbstractDirectoryTestSuite.executeTestSet (AbstractDirectoryTestSuite.java:140) [23] org.apache.maven.surefire.suite.AbstractDirectoryTestSuite.execute (AbstractDirectoryTestSuite.java:165) [24] org.apache.maven.surefire.Surefire.run (Surefire.java:107) [25] sun.reflect.NativeMethodAccessorImpl.invoke0 (native method) [26] sun.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:39) [27] sun.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:37) [28] java.lang.reflect.Method.invoke (Method.java:599) [29] org.apache.maven.surefire.booter.SurefireBooter.runSuitesInProcess (SurefireBooter.java:289) [30] org.apache.maven.surefire.booter.SurefireBooter.main (SurefireBooter.java:1,005) JIT Compilation Thread: Signal Dispatcher: [1] com.ibm.misc.SignalDispatcher.waitForSignal (native method) [2] com.ibm.misc.SignalDispatcher.run (SignalDispatcher.java:54) Gc Slave Thread: Gc Slave Thread: Gc Slave Thread: Finalizer thread: ActiveMQ Scheduler: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] java.util.Timer$TimerImpl.run (Timer.java:230) ActiveMQ Data File Writer: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:167) [3] org.apache.activemq.kaha.impl.async.NIODataFileAppender.processQueue (NIODataFileAppender.java:79) [4] org.apache.activemq.kaha.impl.async.DataFileAppender$2.run (DataFileAppender.java:234) MemoryPoolMXBean notification dispatcher: [1] com.ibm.lang.management.MemoryNotificationThread.processNotificationLoop (native method) [2] com.ibm.lang.management.MemoryNotificationThread.run (MemoryNotificationThread.java:55) OperatingSystemMXBean notification dispatcher: [1] com.ibm.lang.management.OperatingSystemNotificationThread.processNotificationLoop (native method) [2] com.ibm.lang.management.OperatingSystemNotificationThread.run (OperatingSystemNotificationThread.java:39) RMI Scheduler(0): [1] sun.misc.Unsafe.park (native method) [2] java.util.concurrent.locks.LockSupport.parkNanos (LockSupport.java:213) [3] java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos (AbstractQueuedSynchronizer.java:1,974) [4] java.util.concurrent.DelayQueue.take (DelayQueue.java:175) [5] java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take (ScheduledThreadPoolExecutor.java:593) [6] java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take (ScheduledThreadPoolExecutor.java:586) [7] java.util.concurrent.ThreadPoolExecutor.getTask (ThreadPoolExecutor.java:957) [8] java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:917) [9] java.lang.Thread.run (Thread.java:735) RMI TCP Accept-1099: [1] java.net.PlainSocketImpl.socketAccept (native method) [2] java.net.PlainSocketImpl.accept (PlainSocketImpl.java:414) [3] java.net.ServerSocket.implAccept (ServerSocket.java:464) [4] java.net.ServerSocket.accept (ServerSocket.java:432) [5] sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop (TCPTransport.java:381) [6] sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run (TCPTransport.java:353) [7] java.lang.Thread.run (Thread.java:735) RMI TCP Accept-0: [1] java.net.PlainSocketImpl.socketAccept (native method) [2] java.net.PlainSocketImpl.accept (PlainSocketImpl.java:414) [3] java.net.ServerSocket.implAccept (ServerSocket.java:464) [4] java.net.ServerSocket.accept (ServerSocket.java:432) [5] sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop (TCPTransport.java:381) [6] sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run (TCPTransport.java:353) [7] java.lang.Thread.run (Thread.java:735) RMI Reaper: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] java.lang.ref.ReferenceQueue.remove (ReferenceQueue.java:107) [4] java.lang.ref.ReferenceQueue.remove (ReferenceQueue.java:79) [5] sun.rmi.transport.ObjectTable$Reaper.run (ObjectTable.java:350) [6] java.lang.Thread.run (Thread.java:735) GC Daemon: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] sun.misc.GC$Daemon.run (GC.java:112) RMI RenewClean-[10.65.4.113:56702]: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] java.lang.ref.ReferenceQueue.remove (ReferenceQueue.java:107) [4] sun.rmi.transport.DGCClient$EndpointEntry$RenewCleanThread.run (DGCClient.java:528) [5] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport: tcp://localhost/127.0.0.1:61616: [1] java.net.SocketInputStream.socketRead0 (native method) [2] java.net.SocketInputStream.read (SocketInputStream.java:140) [3] org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill (TcpBufferedInputStream.java:50) [4] org.apache.activemq.transport.tcp.TcpBufferedInputStream.read (TcpBufferedInputStream.java:58) [5] java.io.DataInputStream.readInt (DataInputStream.java:381) [6] org.apache.activemq.openwire.OpenWireFormat.unmarshal (OpenWireFormat.java:272) [7] org.apache.activemq.transport.tcp.TcpTransport.readCommand (TcpTransport.java:210) [8] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:202) [9] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [10] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport: tcp://localhost/127.0.0.1:61616: [1] java.net.SocketInputStream.socketRead0 (native method) [2] java.net.SocketInputStream.read (SocketInputStream.java:140) [3] org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill (TcpBufferedInputStream.java:50) [4] org.apache.activemq.transport.tcp.TcpBufferedInputStream.read (TcpBufferedInputStream.java:58) [5] java.io.DataInputStream.readInt (DataInputStream.java:381) [6] org.apache.activemq.openwire.OpenWireFormat.unmarshal (OpenWireFormat.java:272) [7] org.apache.activemq.transport.tcp.TcpTransport.readCommand (TcpTransport.java:210) [8] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:202) [9] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [10] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport: tcp:///127.0.0.1:56744: [1] java.util.HashMap.putAllImpl (HashMap.java:679) [2] java.util.HashMap.<init> (HashMap.java:369) [3] org.apache.activemq.command.Message.copy (Message.java:120) [4] org.apache.activemq.command.ActiveMQMessage.copy (ActiveMQMessage.java:64) [5] org.apache.activemq.command.ActiveMQTextMessage.copy (ActiveMQTextMessage.java:57) [6] org.apache.activemq.command.ActiveMQTextMessage.copy (ActiveMQTextMessage.java:52) [7] org.apache.activemq.broker.region.RegionBroker.sendToDeadLetterQueue (RegionBroker.java:710) [8] org.apache.activemq.broker.region.PrefetchSubscription.sendToDLQ (PrefetchSubscription.java:462) [9] org.apache.activemq.broker.region.PrefetchSubscription.acknowledge (PrefetchSubscription.java:365) [10] org.apache.activemq.broker.region.AbstractRegion.acknowledge (AbstractRegion.java:369) [11] org.apache.activemq.broker.region.RegionBroker.acknowledge (RegionBroker.java:470) [12] org.apache.activemq.broker.TransactionBroker.acknowledge (TransactionBroker.java:195) [13] org.apache.activemq.broker.BrokerFilter.acknowledge (BrokerFilter.java:74) [14] org.apache.activemq.broker.BrokerFilter.acknowledge (BrokerFilter.java:74) [15] org.apache.activemq.broker.MutableBrokerFilter.acknowledge (MutableBrokerFilter.java:85) [16] org.apache.activemq.broker.TransportConnection.processMessageAck (TransportConnection.java:466) [17] org.apache.activemq.command.MessageAck.visit (MessageAck.java:205) [18] org.apache.activemq.broker.TransportConnection.service (TransportConnection.java:309) [19] org.apache.activemq.broker.TransportConnection$1.onCommand (TransportConnection.java:179) [20] org.apache.activemq.transport.TransportFilter.onCommand (TransportFilter.java:68) [21] org.apache.activemq.transport.WireFormatNegotiator.onCommand (WireFormatNegotiator.java:113) [22] org.apache.activemq.transport.InactivityMonitor.onCommand (InactivityMonitor.java:210) [23] org.apache.activemq.transport.TransportSupport.doConsume (TransportSupport.java:84) [24] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:203) [25] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [26] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport: tcp:///127.0.0.1:56733: [1] java.util.HashMap.putAllImpl (HashMap.java:679) [2] java.util.HashMap.<init> (HashMap.java:369) [3] org.apache.activemq.command.Message.copy (Message.java:120) [4] org.apache.activemq.command.ActiveMQMessage.copy (ActiveMQMessage.java:64) [5] org.apache.activemq.command.ActiveMQTextMessage.copy (ActiveMQTextMessage.java:57) [6] org.apache.activemq.command.ActiveMQTextMessage.copy (ActiveMQTextMessage.java:52) [7] org.apache.activemq.broker.region.RegionBroker.sendToDeadLetterQueue (RegionBroker.java:710) [8] org.apache.activemq.broker.region.PrefetchSubscription.sendToDLQ (PrefetchSubscription.java:462) [9] org.apache.activemq.broker.region.PrefetchSubscription.acknowledge (PrefetchSubscription.java:365) [10] org.apache.activemq.broker.region.AbstractRegion.acknowledge (AbstractRegion.java:369) [11] org.apache.activemq.broker.region.RegionBroker.acknowledge (RegionBroker.java:470) [12] org.apache.activemq.broker.TransactionBroker.acknowledge (TransactionBroker.java:195) [13] org.apache.activemq.broker.BrokerFilter.acknowledge (BrokerFilter.java:74) [14] org.apache.activemq.broker.BrokerFilter.acknowledge (BrokerFilter.java:74) [15] org.apache.activemq.broker.MutableBrokerFilter.acknowledge (MutableBrokerFilter.java:85) [16] org.apache.activemq.broker.TransportConnection.processMessageAck (TransportConnection.java:466) [17] org.apache.activemq.command.MessageAck.visit (MessageAck.java:205) [18] org.apache.activemq.broker.TransportConnection.service (TransportConnection.java:309) [19] org.apache.activemq.broker.TransportConnection$1.onCommand (TransportConnection.java:179) [20] org.apache.activemq.transport.TransportFilter.onCommand (TransportFilter.java:68) [21] org.apache.activemq.transport.WireFormatNegotiator.onCommand (WireFormatNegotiator.java:113) [22] org.apache.activemq.transport.InactivityMonitor.onCommand (InactivityMonitor.java:210) [23] org.apache.activemq.transport.TransportSupport.doConsume (TransportSupport.java:84) [24] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:203) [25] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [26] java.lang.Thread.run (Thread.java:735) InactivityMonitor ReadCheck: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] java.util.Timer$TimerImpl.run (Timer.java:230) InactivityMonitor WriteCheck: [1] java.lang.Object.wait (native method) [2] java.lang.Object.wait (Object.java:196) [3] java.util.Timer$TimerImpl.run (Timer.java:230) ActiveMQ Transport: tcp://localhost/127.0.0.1:61616: [1] java.net.SocketInputStream.socketRead0 (native method) [2] java.net.SocketInputStream.read (SocketInputStream.java:140) [3] org.apache.activemq.transport.tcp.TcpBufferedInputStream.fill (TcpBufferedInputStream.java:50) [4] org.apache.activemq.transport.tcp.TcpBufferedInputStream.read (TcpBufferedInputStream.java:58) [5] java.io.DataInputStream.readInt (DataInputStream.java:381) [6] org.apache.activemq.openwire.OpenWireFormat.unmarshal (OpenWireFormat.java:272) [7] org.apache.activemq.transport.tcp.TcpTransport.readCommand (TcpTransport.java:210) [8] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:202) [9] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [10] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport: tcp:///127.0.0.1:56754: [1] org.apache.activemq.broker.region.PrefetchSubscription.dispatchPending (PrefetchSubscription.java:582) [2] org.apache.activemq.broker.region.PrefetchSubscription.add (PrefetchSubscription.java:158) [3] org.apache.activemq.broker.region.DurableTopicSubscription.add (DurableTopicSubscription.java:199) [4] org.apache.activemq.broker.region.policy.SimpleDispatchPolicy.dispatch (SimpleDispatchPolicy.java:49) [5] org.apache.activemq.broker.region.Topic.dispatch (Topic.java:588) [6] org.apache.activemq.broker.region.Topic.doMessageSend (Topic.java:442) [7] org.apache.activemq.broker.region.Topic.send (Topic.java:376) [8] org.apache.activemq.broker.region.AbstractRegion.send (AbstractRegion.java:354) [9] org.apache.activemq.broker.region.RegionBroker.send (RegionBroker.java:445) [10] org.apache.activemq.broker.TransactionBroker.send (TransactionBroker.java:227) [11] org.apache.activemq.broker.BrokerFilter.send (BrokerFilter.java:126) [12] org.apache.activemq.broker.CompositeDestinationBroker.send (CompositeDestinationBroker.java:95) [13] org.apache.activemq.broker.MutableBrokerFilter.send (MutableBrokerFilter.java:133) [14] org.apache.activemq.broker.TransportConnection.processMessage (TransportConnection.java:459) [15] org.apache.activemq.command.ActiveMQMessage.visit (ActiveMQMessage.java:631) [16] org.apache.activemq.broker.TransportConnection.service (TransportConnection.java:309) [17] org.apache.activemq.broker.TransportConnection$1.onCommand (TransportConnection.java:179) [18] org.apache.activemq.transport.TransportFilter.onCommand (TransportFilter.java:68) [19] org.apache.activemq.transport.WireFormatNegotiator.onCommand (WireFormatNegotiator.java:113) [20] org.apache.activemq.transport.InactivityMonitor.onCommand (InactivityMonitor.java:210) [21] org.apache.activemq.transport.TransportSupport.doConsume (TransportSupport.java:84) [22] org.apache.activemq.transport.tcp.TcpTransport.doRun (TcpTransport.java:203) [23] org.apache.activemq.transport.tcp.TcpTransport.run (TcpTransport.java:185) [24] java.lang.Thread.run (Thread.java:735) QueueThread:queue://ActiveMQ.DLQ: [1] sun.misc.Unsafe.park (native method) [2] java.util.concurrent.locks.LockSupport.park (LockSupport.java:173) [3] java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await (AbstractQueuedSynchronizer.java:1,936) [4] java.util.concurrent.LinkedBlockingQueue.take (LinkedBlockingQueue.java:369) [5] java.util.concurrent.ThreadPoolExecutor.getTask (ThreadPoolExecutor.java:957) [6] java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:917) [7] java.lang.Thread.run (Thread.java:735) ActiveMQ Transport Stopper: /127.0.0.1:56733: [1] sun.misc.Unsafe.park (native method) [2] java.util.concurrent.locks.LockSupport.park (LockSupport.java:173) [3] java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt (AbstractQueuedSynchronizer.java:758) [4] java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued (AbstractQueuedSynchronizer.java:789) [5] java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire (AbstractQueuedSynchronizer.java:1,125) [6] java.util.concurrent.locks.ReentrantReadWriteLock$WriteLock.lock (ReentrantReadWriteLock.java:818) [7] org.apache.activemq.broker.TransportConnection$3.run (TransportConnection.java:931) InactivityMonitor Async Task: java.util.concurrent.ThreadPoolExecutor$Worker@4700470: [1] sun.misc.Unsafe.park (native method) [2] java.util.concurrent.locks.LockSupport.parkNanos (LockSupport.java:213) [3] java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill (SynchronousQueue.java:435) [4] java.util.concurrent.SynchronousQueue$TransferStack.transfer (SynchronousQueue.java:334) [5] java.util.concurrent.SynchronousQueue.poll (SynchronousQueue.java:885) [6] java.util.concurrent.ThreadPoolExecutor.getTask (ThreadPoolExecutor.java:955) [7] java.util.concurrent.ThreadPoolExecutor$Worker.run (ThreadPoolExecutor.java:917) [8] java.lang.Thread.run (Thread.java:735)