[AMQ-6686] Timer has cancelled exception in AbstarctInactivityMonitor

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Not A Problem
• Affects Version/s: 5.14.3
• Component/s: Pool

Environment

Integrated activemq-spring with Netty framework and created a PooledConenctionFactory. OS is Linux

• Assignee: Unassigned
• Reporter: Niranjan
• Created: 25/May/17 05:44
• Updated: 25/May/17 13:43
• Resolved: 25/May/17 13:43

Description

Getting below exceptions continuously when i fired around 100 thread to my spring-jms application.Looks like its not able to add new connection in the PooledConnectionFactory even though there is no active connections in the Pool. Check logs below :

javax.jms.JMSException: Error while attempting to add new Connection to the pool
at org.apache.activemq.jms.pool.PooledConnectionFactory.createJmsException(PooledConnectionFactory.java:266) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:225) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:204) ~[cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.messaging.SpringJmsClient.pollAll(SpringJmsClient.java:147) [cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.messaging.MessageBrokerJmsClient.pollAll(MessageBrokerJmsClient.java:191) [cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.handlers.TcpHandler.handleConnecting(TcpHandler.java:272) [cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.handlers.TcpHandler.channelProcessing(TcpHandler.java:172) [cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.handlers.TcpHandler.channelProcessing(TcpHandler.java:161) [cloud-gateway-final.jar:?]
at com.corebrands.bluebolt.handlers.TcpHandler.channelRead(TcpHandler.java:126) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:102) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:287) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) [cloud-gateway-final.jar:?]
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340) [cloud-gateway-final.jar:?]
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1334) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362) [cloud-gateway-final.jar:?]
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348) [cloud-gateway-final.jar:?]
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:926) [cloud-gateway-final.jar:?]
at io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:1017) [cloud-gateway-final.jar:?]
at io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:394) [cloud-gateway-final.jar:?]
at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:299) [cloud-gateway-final.jar:?]
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:858) [cloud-gateway-final.jar:?]
at io.netty.util.concurrent.DefaultThreadFactory$DefaultRunnableDecorator.run(DefaultThreadFactory.java:144) [cloud-gateway-final.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_131]
Caused by: javax.jms.JMSException: Could not connect to broker URL: tcp://ec2-32-57-451-65.us-west-2.compute.amazonaws.com:61616?wireFormat.maxInactivityDuration=0. Reason: java.lang.IllegalStateException: Timer already cancelled.
at org.apache.activemq.util.JMSExceptionSupport.create(JMSExceptionSupport.java:36) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:374) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:304) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:244) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:275) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:95) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:92) ~[cloud-gateway-final.jar:?]
at org.apache.commons.pool2.impl.GenericKeyedObjectPool.create(GenericKeyedObjectPool.java:1012) ~[cloud-gateway-final.jar:?]
at org.apache.commons.pool2.impl.GenericKeyedObjectPool.addObject(GenericKeyedObjectPool.java:1192) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:221) ~[cloud-gateway-final.jar:?]
... 51 more
Caused by: java.lang.IllegalStateException: Timer already cancelled.
at java.util.Timer.sched(Timer.java:397) ~[?:1.8.0_131]
at java.util.Timer.schedule(Timer.java:193) ~[?:1.8.0_131]
at org.apache.activemq.transport.AbstractInactivityMonitor.startConnectCheckTask(AbstractInactivityMonitor.java:425) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.AbstractInactivityMonitor.startConnectCheckTask(AbstractInactivityMonitor.java:400) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.InactivityMonitor.start(InactivityMonitor.java:50) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:64) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.WireFormatNegotiator.start(WireFormatNegotiator.java:72) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:64) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.transport.TransportFilter.start(TransportFilter.java:64) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:354) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createActiveMQConnection(ActiveMQConnectionFactory.java:304) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.ActiveMQConnectionFactory.createConnection(ActiveMQConnectionFactory.java:244) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:275) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:95) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory$1.makeObject(PooledConnectionFactory.java:92) ~[cloud-gateway-final.jar:?]
at org.apache.commons.pool2.impl.GenericKeyedObjectPool.create(GenericKeyedObjectPool.java:1012) ~[cloud-gateway-final.jar:?]
at org.apache.commons.pool2.impl.GenericKeyedObjectPool.addObject(GenericKeyedObjectPool.java:1192) ~[cloud-gateway-final.jar:?]
at org.apache.activemq.jms.pool.PooledConnectionFactory.createConnection(PooledConnectionFactory.java:221) ~[cloud-gateway-final.jar:?]
... 51 more