Details
-
Bug
-
Status: Closed
-
Critical
-
Resolution: Abandoned
-
5.13.1
-
None
-
None
Description
system is horizontally scaled with 2 Active MQ nodes.
It appears to be on Node 2 only, the ActiveMQ service is frequently stopped by the system.
We see some SQL connectivity issue in the logs, around the same time frame. But we do not see ActiveMQ on node 1 is being affected.
Its the Node 2 ActiveMQ that is stopping.
<persistenceAdapter>
<jdbcPersistenceAdapter dataDirectory="activemq-data" dataSource="#mssql-ds" lockKeepAlivePeriod="0"/>
</persistenceAdapter>
<ioExceptionHandler>
<defaultIOExceptionHandler ignoreSQLExceptions="false" stopStartConnectors="true" />
</ioExceptionHandler>
While restarting transports
java.lang.NullPointerException
INFO | jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.util.DefaultIOExceptionHandler$1$1.run(DefaultIOExceptionHandler.java:114)[
follow below logs to see exception details exactly...
=====================================
| jvm 1 | 2016/11/23 01:13:01 | ERROR | Failed to start Apache ActiveMQ ([localhost, null], org.apache.activemq.broker.SuppressReplyException: Stop/RestartTransportsInitiated) INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Connector openwire stopped INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Connector amqp stopped INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Successfully stopped transports on BrokerService[localhost] INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Apache ActiveMQ 5.13.1 (localhost, null) is shutting down INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Connector openwire stopped INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Connector amqp stopped INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | PListStore:[E:\NCRApps\apache-activemq-5.13.1\bin\win64\..\..\data\localhost\tmp_storage] stopped INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Apache ActiveMQ 5.13.1 (localhost, null) uptime 10 hours 55 minutes INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Apache ActiveMQ 5.13.1 (localhost, null) is shutdown INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Closing org.apache.activemq.xbean.XBeanBrokerFactory$1@6803ef71: startup date [Tue Nov 22 14:17:56 EST 2016]; root of context hierarchy INFO |
jvm 1 | 2016/11/23 01:13:01 | WARN | Exception thrown from LifecycleProcessor on context close INFO |
jvm 1 | 2016/11/23 01:13:01 | java.lang.IllegalStateException: LifecycleProcessor not initialized - call 'refresh' before invoking lifecycle methods via the context: org.apache.activemq.xbean.XBeanBrokerFactory$1@6803ef71: startup date [Tue Nov 22 14:17:56 EST 2016]; root of context hierarchy INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.getLifecycleProcessor(AbstractApplicationContext.java:357)[spring-context-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.doClose(AbstractApplicationContext.java:884)[spring-context-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.close(AbstractApplicationContext.java:843)[spring-context-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.hooks.SpringContextHook.run(SpringContextHook.java:30)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerService.stop(BrokerService.java:873)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerService.stop(XBeanBrokerService.java:122)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerService.start(BrokerService.java:627)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerService.afterPropertiesSet(XBeanBrokerService.java:73)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeCustomInitMethod(AbstractAutowireCapableBeanFactory.java:1700)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.invokeInitMethods(AbstractAutowireCapableBeanFactory.java:1639)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.initializeBean(AbstractAutowireCapableBeanFactory.java:1568)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:539)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:476)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractBeanFactory$1.getObject(AbstractBeanFactory.java:303)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:230)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:299)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:194)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:755)[spring-beans-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:762)[spring-context-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:480)[spring-context-4.1.9.RELEASE.jar:4.1.9.RELEASE] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64)[xbean-spring-3.18.jar:3.18] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52)[xbean-spring-3.18.jar:3.18] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67)[activemq-spring-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87)[activemq-console-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63)[activemq-console-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:154)[activemq-console-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63)[activemq-console-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104)[activemq-console-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.runTaskClass(Main.java:262)[activemq.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.main(Main.java:115)[activemq.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.tanukisoftware.wrapper.WrapperSimpleApp.run(WrapperSimpleApp.java:240)[wrapper.jar:3.2.3] INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.Thread.run(Unknown Source)[:1.8.0_66] INFO |
jvm 1 | 2016/11/23 01:13:01 | WARN | Stopping BrokerService[localhost] due to failure restarting transports INFO |
jvm 1 | 2016/11/23 01:13:01 | java.lang.NullPointerException INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.util.DefaultIOExceptionHandler$1$1.run(DefaultIOExceptionHandler.java:114)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | INFO | Stopping BrokerService[localhost] due to exception, java.lang.NullPointerException INFO |
jvm 1 | 2016/11/23 01:13:01 | java.lang.NullPointerException INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.util.DefaultIOExceptionHandler$1$1.run(DefaultIOExceptionHandler.java:114)[activemq-broker-5.13.1.jar:5.13.1] INFO |
jvm 1 | 2016/11/23 01:13:01 | WARN | Exception encountered during context initialization - cancelling refresh attempt: org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'org.apache.activemq.xbean.XBeanBrokerService#0' defined in class path resource [activemq.xml]: Invocation of init method failed; nested exception is org.apache.activemq.broker.SuppressReplyException: Stop/RestartTransportsInitiated INFO |
jvm 1 | 2016/11/23 01:13:01 | ERROR: java.lang.RuntimeException: Failed to execute start task. Reason: java.lang.IllegalStateException: BeanFactory not initialized or already closed - call 'refresh' before accessing beans via the ApplicationContext INFO |
jvm 1 | 2016/11/23 01:13:01 | java.lang.RuntimeException: Failed to execute start task. Reason: java.lang.IllegalStateException: BeanFactory not initialized or already closed - call 'refresh' before accessing beans via the ApplicationContext INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:91) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:154) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.runTaskClass(Main.java:262) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.main(Main.java:115) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.tanukisoftware.wrapper.WrapperSimpleApp.run(WrapperSimpleApp.java:240) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.Thread.run(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | Caused by: java.lang.IllegalStateException: BeanFactory not initialized or already closed - call 'refresh' before accessing beans via the ApplicationContext INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractRefreshableApplicationContext.getBeanFactory(AbstractRefreshableApplicationContext.java:170) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.destroyBeans(AbstractApplicationContext.java:915) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:493) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87) INFO |
jvm 1 | 2016/11/23 01:13:01 | ... 16 more INFO |
jvm 1 | 2016/11/23 01:13:01 | ERROR: java.lang.IllegalStateException: BeanFactory not initialized or already closed - call 'refresh' before accessing beans via the ApplicationContext INFO |
jvm 1 | 2016/11/23 01:13:01 | java.lang.IllegalStateException: BeanFactory not initialized or already closed - call 'refresh' before accessing beans via the ApplicationContext INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractRefreshableApplicationContext.getBeanFactory(AbstractRefreshableApplicationContext.java:170) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.destroyBeans(AbstractApplicationContext.java:915) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:493) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:64) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.xbean.spring.context.ResourceXmlApplicationContext.<init>(ResourceXmlApplicationContext.java:52) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory$1.<init>(XBeanBrokerFactory.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createApplicationContext(XBeanBrokerFactory.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.xbean.XBeanBrokerFactory.createBroker(XBeanBrokerFactory.java:67) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:71) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.broker.BrokerFactory.createBroker(BrokerFactory.java:54) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.StartCommand.runTask(StartCommand.java:87) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.runTask(ShellCommand.java:154) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.AbstractCommand.execute(AbstractCommand.java:63) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.command.ShellCommand.main(ShellCommand.java:104) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.runTaskClass(Main.java:262) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.apache.activemq.console.Main.main(Main.java:115) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at sun.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.reflect.Method.invoke(Unknown Source) INFO |
jvm 1 | 2016/11/23 01:13:01 | at org.tanukisoftware.wrapper.WrapperSimpleApp.run(WrapperSimpleApp.java:240) INFO |
jvm 1 | 2016/11/23 01:13:01 | at java.lang.Thread.run(Unknown Source) ======================= |