Details
Major Description
We have a problem related a Leak and the a possible Deadlock, we know that we are using versions outdated. We are in production with this version and we need know the correct version to upgrade where the bug is fixed. Or a new bug found?
We found this bugs: Leak: AMQ-3214 - Deadlock: AMQ-4274
LEAK:
Exception in thread "ActiveMQ Transport: tcp:///10.80.30.150:60721" java.lang.OutOfMemoryError: getNewTla
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:211)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "InactivityMonitor Async Task: java.util.concurrent.ThreadPoolExecutor$Worker@1ff1bec3" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.locks.AbstractQueuedSynchronizer.addWaiter(AbstractQueuedSynchronizer.java:584)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)
at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)
at java.util.concurrent.ThreadPoolExecutor.workerCanExit(ThreadPoolExecutor.java:971)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:950)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "ActiveMQ Session Task-11124" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.CopyOnWriteArrayList.iterator(CopyOnWriteArrayList.java:925)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:190)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "InactivityMonitor Async Task: java.util.concurrent.ThreadPoolExecutor$Worker@1ff1bbd6" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.locks.AbstractQueuedSynchronizer.addWaiter(AbstractQueuedSynchronizer.java:584)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)
at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)
at java.util.concurrent.ThreadPoolExecutor.workerCanExit(ThreadPoolExecutor.java:971)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:950)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "ActiveMQ Session Task-11122" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.CopyOnWriteArrayList.iterator(CopyOnWriteArrayList.java:925)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:190)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "ActiveMQ Session Task-11123" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.CopyOnWriteArrayList.iterator(CopyOnWriteArrayList.java:925)
at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:190)
at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "InactivityMonitor Async Task: java.util.concurrent.ThreadPoolExecutor$Worker@1ff1b3e5" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.locks.AbstractQueuedSynchronizer.addWaiter(AbstractQueuedSynchronizer.java:584)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)
at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)
at java.util.concurrent.ThreadPoolExecutor.workerCanExit(ThreadPoolExecutor.java:971)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:950)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:662)
Exception in thread "InactivityMonitor Async Task: java.util.concurrent.ThreadPoolExecutor$Worker@1ff1bb00" java.lang.OutOfMemoryError: getNewTla
at java.util.concurrent.locks.AbstractQueuedSynchronizer.addWaiter(AbstractQueuedSynchronizer.java:584)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)
at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)
at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:262)
at java.util.concurrent.ThreadPoolExecutor.workerCanExit(ThreadPoolExecutor.java:971)
at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:950)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:907)
at java.lang.Thread.run(Thread.java:662)
DEADLOCK:
Chain 1:
"Camel (MainframeAdapterContext) thread #3 - JmsConsumer[bam_input_temp]" id=25 idx=0xa4 tid=6373 waiting for java/lang/Object@0x820d0a98 held by:
"Camel (MainframeAdapterContext) thread #25 - JmsConsumer[bam_input_temp]" id=47 idx=0xfc tid=6399 waiting for java/util/concurrent/locks/ReentrantLock$NonfairSync@0x824288d8 held by:
"Camel (MainframeAdapterContext) thread #92 - JmsConsumer[bam_input_temp]" id=114 idx=0x208 tid=6496 (waiting on notification)
"Camel (MainframeAdapterContext) thread #3 - JmsConsumer[bam_input_temp]" id=25 idx=0xa4 tid=6373 prio=5 alive, blocked, native_blocked, daemon
– Blocked trying to get lock: java/lang/Object@0x820d0a98[fat lock]
at jrockit/vm/Threads.waitForUnblockSignal()V(Native Method)
at jrockit/vm/Locks.fatLockBlockOrSpin(Locks.java:1411)[optimized]
at jrockit/vm/Locks.lockFat(Locks.java:1512)[optimized]
at jrockit/vm/Locks.monitorEnterSecondStageHard(Locks.java:1054)[optimized]
at jrockit/vm/Locks.monitorEnterSecondStage(Locks.java:1005)[optimized]
at jrockit/vm/Locks.monitorEnter(Locks.java:2179)[optimized]
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.clearResources(DefaultMessageListenerContainer.java:1097)
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:958)
at java/util/concurrent/ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java/util/concurrent/ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java/lang/Thread.run(Thread.java:662)
at jrockit/vm/RNI.c2java(JJJJJ)V(Native Method)
– end of trace
"Camel (MainframeAdapterContext) thread #25 - JmsConsumer[bam_input_temp]" id=47 idx=0xfc tid=6399 prio=5 alive, parked, native_blocked, daemon
– Parking to wait for: java/util/concurrent/locks/ReentrantLock$NonfairSync@0x824288d8
at jrockit/vm/Locks.park0(J)V(Native Method)
at jrockit/vm/Locks.park(Locks.java:2230)[inlined]
at jrockit/proxy/sun/misc/Unsafe.park(Unsafe.java:616)[inlined]
at java/util/concurrent/locks/LockSupport.park(LockSupport.java:156)[inlined]
at java/util/concurrent/locks/AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:811)[inlined]
at java/util/concurrent/locks/AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:842)[inlined]
at java/util/concurrent/locks/AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1178)[inlined]
at java/util/concurrent/locks/ReentrantLock$NonfairSync.lock(ReentrantLock.java:186)[optimized]
at java/util/concurrent/locks/ReentrantLock.lock(ReentrantLock.java:262)[inlined]
at org/apache/activemq/transport/MutexTransport.oneway(MutexTransport.java:66)[optimized]
at org/apache/activemq/transport/ResponseCorrelator.oneway(ResponseCorrelator.java:60)[optimized]
at org/apache/activemq/ActiveMQConnection.doAsyncSendPacket(ActiveMQConnection.java:1275)[inlined]
at org/apache/activemq/ActiveMQConnection.asyncSendPacket(ActiveMQConnection.java:1269)[optimized]
at org/apache/activemq/ActiveMQSession.doClose(ActiveMQSession.java:644)
at org/apache/activemq/ActiveMQSession.close(ActiveMQSession.java:634)
at org/apache/activemq/pool/SessionPool.destroyObject(SessionPool.java:91)
at org/apache/commons/pool/impl/GenericObjectPool.destroy(GenericObjectPool.java:1330)
at org/apache/commons/pool/impl/GenericObjectPool.clear(GenericObjectPool.java:1315)
at org/apache/commons/pool/impl/GenericObjectPool.close(GenericObjectPool.java:1482)
^-- Holding lock: org/apache/commons/pool/impl/GenericObjectPool@0x8220a8b0[thin lock]
at org/apache/activemq/pool/SessionPool.close(SessionPool.java:47)
at org/apache/activemq/pool/ConnectionPool.close(ConnectionPool.java:133)
^-- Holding lock: org/apache/activemq/pool/ConnectionPool@0x8220a858[recursive]
at org/apache/activemq/pool/ConnectionPool.expiredCheck(ConnectionPool.java:184)[optimized]
^-- Holding lock: org/apache/activemq/pool/ConnectionPool@0x8220a858[recursive]
at org/apache/activemq/pool/ConnectionPool.decrementReferenceCount(ConnectionPool.java:157)[inlined]
at org/apache/activemq/pool/PooledConnection.close(PooledConnection.java:80)[optimized]
^-- Holding lock: org/apache/activemq/pool/ConnectionPool@0x8220a858[thin lock]
at org/springframework/jms/connection/ConnectionFactoryUtils.releaseConnection(ConnectionFactoryUtils.java:80)[optimized]
at org/springframework/jms/listener/AbstractJmsListeningContainer.refreshSharedConnection(AbstractJmsListeningContainer.java:385)
^-- Holding lock: java/lang/Object@0x820d0a98[fat lock]
at org/springframework/jms/listener/DefaultMessageListenerContainer.refreshConnectionUntilSuccessful(DefaultMessageListenerContainer.java:856)
at org/springframework/jms/listener/DefaultMessageListenerContainer.recoverAfterListenerSetupFailure(DefaultMessageListenerContainer.java:838)
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:969)
^-- Holding lock: java/lang/Object@0x820d0b18[thin lock]
at java/util/concurrent/ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java/util/concurrent/ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java/lang/Thread.run(Thread.java:662)
at jrockit/vm/RNI.c2java(JJJJJ)V(Native Method)
– end of trace
"Camel (MainframeAdapterContext) thread #92 - JmsConsumer[bam_input_temp]" id=114 idx=0x208 tid=6496 prio=5 alive, waiting, native_blocked, daemon
– Waiting for notification on: java/lang/Object@0x822fc808[fat lock]
at jrockit/vm/Threads.waitForNotifySignal(JLjava/lang/Object;)Z(Native Method)
at jrockit/vm/Locks.wait(Locks.java:1973)[inlined]
at org/apache/activemq/transport/failover/FailoverTransport.oneway(FailoverTransport.java:562)[optimized]
^-- Lock released while waiting: java/lang/Object@0x822fc808[fat lock]
at org/apache/activemq/transport/MutexTransport.oneway(MutexTransport.java:68)[optimized]
at org/apache/activemq/transport/ResponseCorrelator.oneway(ResponseCorrelator.java:60)[optimized]
at org/apache/activemq/ActiveMQConnection.doAsyncSendPacket(ActiveMQConnection.java:1275)[inlined]
at org/apache/activemq/ActiveMQConnection.asyncSendPacket(ActiveMQConnection.java:1269)[inlined]
at org/apache/activemq/ActiveMQSession.send(ActiveMQSession.java:1769)[inlined]
at org/apache/activemq/ActiveMQMessageProducer.send(ActiveMQMessageProducer.java:277)[inlined]
at org/apache/activemq/ActiveMQMessageProducer.send(ActiveMQMessageProducer.java:212)[inlined]
at org/apache/activemq/pool/PooledProducer.send(PooledProducer.java:74)[inlined]
at org/apache/activemq/pool/PooledProducer.send(PooledProducer.java:63)[optimized]
^-- Holding lock: java/lang/Object@0x81a712d0[thin lock]
^-- Holding lock: org/apache/activemq/ActiveMQMessageProducer@0x820de070[biased lock]
at org/springframework/jms/core/JmsTemplate.doSend(JmsTemplate.java:589)[inlined]
at org/apache/camel/component/jms/JmsConfiguration$CamelJmsTemplate.doSend(JmsConfiguration.java:277)[optimized]
at org/apache/camel/component/jms/JmsConfiguration$CamelJmsTemplate.doSendToDestination(JmsConfiguration.java:216)[inlined]
at org/apache/camel/component/jms/JmsConfiguration$CamelJmsTemplate.access$100(JmsConfiguration.java:158)[inlined]
at org/apache/camel/component/jms/JmsConfiguration$CamelJmsTemplate$3.doInJms(JmsConfiguration.java:192)[optimized]
at org/springframework/jms/core/JmsTemplate.execute(JmsTemplate.java:466)[inlined]
at org/apache/camel/component/jms/JmsConfiguration$CamelJmsTemplate.send(JmsConfiguration.java:189)[inlined]
at org/apache/camel/component/jms/JmsProducer.doSend(JmsProducer.java:398)[inlined]
at org/apache/camel/component/jms/JmsProducer.processInOnly(JmsProducer.java:352)[inlined]
at org/apache/camel/component/jms/JmsProducer.process(JmsProducer.java:132)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)[optimized]
at org/apache/camel/processor/RoutingSlip$2.doInAsyncProducer(RoutingSlip.java:287)[inlined]
at org/apache/camel/impl/ProducerCache.doInAsyncProducer(ProducerCache.java:298)[inlined]
at org/apache/camel/processor/RoutingSlip.processExchange(RoutingSlip.java:280)[inlined]
at org/apache/camel/processor/RoutingSlip.doRoutingSlip(RoutingSlip.java:205)[inlined]
at org/apache/camel/processor/RoutingSlip.process(RoutingSlip.java:135)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[inlined]
at org/apache/camel/management/InstrumentationProcessor.process(InstrumentationProcessor.java:73)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)[optimized]
at org/apache/camel/processor/DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[inlined]
at org/apache/camel/processor/interceptor/TraceInterceptor.process(TraceInterceptor.java:91)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)[inlined]
at org/apache/camel/processor/RedeliveryErrorHandler.processErrorHandler(RedeliveryErrorHandler.java:334)[inlined]
at org/apache/camel/processor/RedeliveryErrorHandler.process(RedeliveryErrorHandler.java:220)[optimized]
at org/apache/camel/processor/RouteContextProcessor.processNext(RouteContextProcessor.java:45)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[inlined]
at org/apache/camel/processor/interceptor/DefaultChannel.process(DefaultChannel.java:303)[optimized]
at org/apache/camel/processor/RouteContextProcessor.processNext(RouteContextProcessor.java:45)[optimized]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[optimized]
at org/apache/camel/processor/UnitOfWorkProcessor.processAsync(UnitOfWorkProcessor.java:150)[inlined]
at org/apache/camel/processor/UnitOfWorkProcessor.process(UnitOfWorkProcessor.java:117)[optimized]
at org/apache/camel/processor/RouteInflightRepositoryProcessor.processNext(RouteInflightRepositoryProcessor.java:48)[optimized]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:73)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.processNext(DelegateAsyncProcessor.java:99)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:90)[inlined]
at org/apache/camel/management/InstrumentationProcessor.process(InstrumentationProcessor.java:73)[optimized]
at org/apache/camel/util/AsyncProcessorHelper.process(AsyncProcessorHelper.java:99)[inlined]
at org/apache/camel/processor/DelegateAsyncProcessor.process(DelegateAsyncProcessor.java:86)[optimized]
at org/apache/camel/component/jms/EndpointMessageListener.onMessage(EndpointMessageListener.java:104)[optimized]
at org/springframework/jms/listener/AbstractMessageListenerContainer.doInvokeListener(AbstractMessageListenerContainer.java:560)[inlined]
at org/springframework/jms/listener/AbstractMessageListenerContainer.invokeListener(AbstractMessageListenerContainer.java:498)[inlined]
at org/springframework/jms/listener/AbstractMessageListenerContainer.doExecuteListener(AbstractMessageListenerContainer.java:467)[optimized]
at org/springframework/jms/listener/AbstractPollingMessageListenerContainer.doReceiveAndExecute(AbstractPollingMessageListenerContainer.java:325)[optimized]
at org/springframework/jms/listener/AbstractPollingMessageListenerContainer.receiveAndExecute(AbstractPollingMessageListenerContainer.java:263)[inlined]
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.invokeListener(DefaultMessageListenerContainer.java:1058)[optimized]
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.executeOngoingLoop(DefaultMessageListenerContainer.java:1050)
at org/springframework/jms/listener/DefaultMessageListenerContainer$AsyncMessageListenerInvoker.run(DefaultMessageListenerContainer.java:947)
at java/util/concurrent/ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java/util/concurrent/ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java/lang/Thread.run(Thread.java:662)
at jrockit/vm/RNI.c2java(JJJJJ)V(Native Method)
– end of trace
Attachments
Issue Links
- contains
-
AMQ-3588 PooledSession - Can cause memory leak in case many consumers is created by the session
-
- Resolved
-