[AMQ-4911] Activemq running on standalone not able to to post messages to database in case of database failover

• Type: Bug
• Status: Closed
• Priority: Major
• Resolution: Abandoned
• Affects Version/s: 5.8.0
• Component/s: Broker

Environment

activemq 5.8 with oracle RAC with 2 nodes. activemq in windows platform

• Assignee: Unassigned
• Reporter: kishan
• Created: 01/Dec/13 06:34
• Updated: 13/Mar/22 14:16
• Resolved: 13/Mar/22 14:16

Description

when both the database nodes were up and running things were working fine with active mq , then after closing the database server on one of the nodes say node2, things some how worked good and messages were posted in the database, but when brought back node2 and brought node1 down, messages went to queued state and message finally were lost with these exceptions, but after 5 -6mins things were stablized again

2013-11-28 11:04:09,515 | WARN | Error while closing connection: No more data to read from socket, due to: No more data to read from socket | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Transport: tcp:///10.167.91.198:58115@61618
java.sql.SQLException: No more data to read from socket
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1123)
at oracle.jdbc.driver.T4CMAREngine.unmarshalSB1(T4CMAREngine.java:1075)
at oracle.jdbc.driver.T4C8Oall.receive(T4C8Oall.java:480)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:219)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:970)
at oracle.jdbc.driver.OraclePreparedStatement.executeBatch(OraclePreparedStatement.java:10690)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:106)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:84)
at org.apache.activemq.store.jdbc.TransactionContext.close(TransactionContext.java:132)
at org.apache.activemq.store.jdbc.JDBCMessageStore.addMessage(JDBCMessageStore.java:129)
at org.apache.activemq.store.memory.MemoryTransactionStore.addMessage(MemoryTransactionStore.java:327)
at org.apache.activemq.store.memory.MemoryTransactionStore$1.asyncAddQueueMessage(MemoryTransactionStore.java:154)
at org.apache.activemq.broker.region.Queue.doMessageSend(Queue.java:748)
at org.apache.activemq.broker.region.Queue.send(Queue.java:721)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:406)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:392)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.scheduler.SchedulerBroker.send(SchedulerBroker.java:177)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:317)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:499)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:749)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:329)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:288)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 11:04:26,036 | WARN | Old message cleanup failed due to: java.sql.SQLException: OALL8 is in an inconsistent state | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
2013-11-28 11:04:26,036 | WARN | Failure Details: OALL8 is in an inconsistent state | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
java.sql.SQLException: OALL8 is in an inconsistent state
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4C8Oall.marshal(T4C8Oall.java:352)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:205)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:970)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1190)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3370)
at oracle.jdbc.driver.OraclePreparedStatement.executeUpdate(OraclePreparedStatement.java:3454)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doDeleteOldMessages(DefaultJDBCAdapter.java:820)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter.cleanup(JDBCPersistenceAdapter.java:330)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter$3.run(JDBCPersistenceAdapter.java:311)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 11:09:26,040 | WARN | Old message cleanup failed due to: java.sql.SQLException: Io exception: Software caused connection abort: socket write error | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
2013-11-28 11:09:26,040 | WARN | Failure Details: Io exception: Software caused connection abort: socket write error | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
java.sql.SQLException: Io exception: Software caused connection abort: socket write error
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:255)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:992)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1190)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3370)
at oracle.jdbc.driver.OraclePreparedStatement.executeUpdate(OraclePreparedStatement.java:3454)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doDeleteOldMessages(DefaultJDBCAdapter.java:820)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter.cleanup(JDBCPersistenceAdapter.java:330)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter$3.run(JDBCPersistenceAdapter.java:311)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 11:09:26,040 | WARN | Close failed: Already closed. | org.apache.activemq.store.jdbc.TransactionContext | ActiveMQ Cleanup Timer
java.sql.SQLException: Already closed.
at org.apache.commons.dbcp.PoolableConnection.close(PoolableConnection.java:114)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.close(PoolingDataSource.java:191)
at org.apache.activemq.store.jdbc.TransactionContext.close(TransactionContext.java:145)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter.cleanup(JDBCPersistenceAdapter.java:339)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter$3.run(JDBCPersistenceAdapter.java:311)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 13:40:10,811 | WARN | Network connection between vm://Node0#0 and tcp:///10.131.90.156:62018@61618 shutdown due to a remote error: java.net.SocketException: Connection reset | org.apache.activemq.network.DemandForwardingBridgeSupport | ActiveMQ Transport: tcp:///10.131.90.156:62018@61618
2013-11-28 13:40:10,827 | INFO | Connector vm://Node0 Stopped | org.apache.activemq.broker.TransportConnector | ActiveMQ BrokerService[Node0] Task-1069
2013-11-28 13:40:10,827 | INFO | Node0 bridge to Node2 stopped | org.apache.activemq.network.DemandForwardingBridgeSupport | ActiveMQ BrokerService[Node0] Task-1069
2013-11-28 17:44:08,692 | WARN | Error while closing connection: No more data to read from socket, due to: No more data to read from socket | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Transport: tcp:///10.131.91.198:50792@61618
java.sql.SQLException: No more data to read from socket
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1123)
at oracle.jdbc.driver.T4CMAREngine.unmarshalSB1(T4CMAREngine.java:1075)
at oracle.jdbc.driver.T4C8Oall.receive(T4C8Oall.java:480)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:219)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:970)
at oracle.jdbc.driver.OraclePreparedStatement.executeBatch(OraclePreparedStatement.java:10690)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:106)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:84)
at org.apache.activemq.store.jdbc.TransactionContext.close(TransactionContext.java:132)
at org.apache.activemq.store.jdbc.JDBCMessageStore.addMessage(JDBCMessageStore.java:129)
at org.apache.activemq.store.memory.MemoryTransactionStore.addMessage(MemoryTransactionStore.java:327)
at org.apache.activemq.store.memory.MemoryTransactionStore$1.asyncAddQueueMessage(MemoryTransactionStore.java:154)
at org.apache.activemq.broker.region.Queue.doMessageSend(Queue.java:748)
at org.apache.activemq.broker.region.Queue.send(Queue.java:721)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:406)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:392)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.scheduler.SchedulerBroker.send(SchedulerBroker.java:177)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:317)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:499)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:749)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:329)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:288)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 17:44:27,163 | WARN | Old message cleanup failed due to: java.sql.SQLException: OALL8 is in an inconsistent state | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
2013-11-28 17:44:27,163 | WARN | Failure Details: OALL8 is in an inconsistent state | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
java.sql.SQLException: OALL8 is in an inconsistent state
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4C8Oall.marshal(T4C8Oall.java:352)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:205)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:970)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1190)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3370)
at oracle.jdbc.driver.OraclePreparedStatement.executeUpdate(OraclePreparedStatement.java:3454)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doDeleteOldMessages(DefaultJDBCAdapter.java:820)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter.cleanup(JDBCPersistenceAdapter.java:330)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter$3.run(JDBCPersistenceAdapter.java:311)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 17:45:55,474 | WARN | Error while closing connection: Io exception: Software caused connection abort: socket write error, due to: Io exception: Software caused connection abort: socket write error | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Transport: tcp:///10.167.91.198:50906@61618
java.sql.SQLException: Io exception: Software caused connection abort: socket write error
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:255)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:992)
at oracle.jdbc.driver.OraclePreparedStatement.executeBatch(OraclePreparedStatement.java:10690)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.commons.dbcp.DelegatingStatement.executeBatch(DelegatingStatement.java:297)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:106)
at org.apache.activemq.store.jdbc.TransactionContext.executeBatch(TransactionContext.java:84)
at org.apache.activemq.store.jdbc.TransactionContext.close(TransactionContext.java:132)
at org.apache.activemq.store.jdbc.JDBCMessageStore.addMessage(JDBCMessageStore.java:129)
at org.apache.activemq.store.memory.MemoryTransactionStore.addMessage(MemoryTransactionStore.java:327)
at org.apache.activemq.store.memory.MemoryTransactionStore$1.asyncAddQueueMessage(MemoryTransactionStore.java:154)
at org.apache.activemq.broker.region.Queue.doMessageSend(Queue.java:748)
at org.apache.activemq.broker.region.Queue.send(Queue.java:721)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:406)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:392)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.scheduler.SchedulerBroker.send(SchedulerBroker.java:177)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:317)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:499)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:749)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:329)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:288)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 17:45:55,474 | WARN | Close failed: Already closed. | org.apache.activemq.store.jdbc.TransactionContext | ActiveMQ Transport: tcp:///10.131.91.198:50906@61618
java.sql.SQLException: Already closed.
at org.apache.commons.dbcp.PoolableConnection.close(PoolableConnection.java:114)
at org.apache.commons.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.close(PoolingDataSource.java:191)
at org.apache.activemq.store.jdbc.TransactionContext.close(TransactionContext.java:145)
at org.apache.activemq.store.jdbc.JDBCMessageStore.addMessage(JDBCMessageStore.java:129)
at org.apache.activemq.store.memory.MemoryTransactionStore.addMessage(MemoryTransactionStore.java:327)
at org.apache.activemq.store.memory.MemoryTransactionStore$1.asyncAddQueueMessage(MemoryTransactionStore.java:154)
at org.apache.activemq.broker.region.Queue.doMessageSend(Queue.java:748)
at org.apache.activemq.broker.region.Queue.send(Queue.java:721)
at org.apache.activemq.broker.region.AbstractRegion.send(AbstractRegion.java:406)
at org.apache.activemq.broker.region.RegionBroker.send(RegionBroker.java:392)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.scheduler.SchedulerBroker.send(SchedulerBroker.java:177)
at org.apache.activemq.broker.BrokerFilter.send(BrokerFilter.java:129)
at org.apache.activemq.broker.CompositeDestinationBroker.send(CompositeDestinationBroker.java:96)
at org.apache.activemq.broker.TransactionBroker.send(TransactionBroker.java:317)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.MutableBrokerFilter.send(MutableBrokerFilter.java:135)
at org.apache.activemq.broker.TransportConnection.processMessage(TransportConnection.java:499)
at org.apache.activemq.command.ActiveMQMessage.visit(ActiveMQMessage.java:749)
at org.apache.activemq.broker.TransportConnection.service(TransportConnection.java:329)
at org.apache.activemq.broker.TransportConnection$1.onCommand(TransportConnection.java:184)
at org.apache.activemq.transport.MutexTransport.onCommand(MutexTransport.java:50)
at org.apache.activemq.transport.WireFormatNegotiator.onCommand(WireFormatNegotiator.java:113)
at org.apache.activemq.transport.AbstractInactivityMonitor.onCommand(AbstractInactivityMonitor.java:288)
at org.apache.activemq.transport.TransportSupport.doConsume(TransportSupport.java:83)
at org.apache.activemq.transport.tcp.TcpTransport.doRun(TcpTransport.java:214)
at org.apache.activemq.transport.tcp.TcpTransport.run(TcpTransport.java:196)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 17:49:27,167 | WARN | Old message cleanup failed due to: java.sql.SQLException: No more data to read from socket | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
2013-11-28 17:49:27,167 | WARN | Failure Details: No more data to read from socket | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Cleanup Timer
java.sql.SQLException: No more data to read from socket
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1123)
at oracle.jdbc.driver.T4CMAREngine.unmarshalSB1(T4CMAREngine.java:1075)
at oracle.jdbc.driver.T4C8Oall.receive(T4C8Oall.java:480)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:219)
at oracle.jdbc.driver.T4CPreparedStatement.executeForRows(T4CPreparedStatement.java:970)
at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1190)
at oracle.jdbc.driver.OraclePreparedStatement.executeInternal(OraclePreparedStatement.java:3370)
at oracle.jdbc.driver.OraclePreparedStatement.executeUpdate(OraclePreparedStatement.java:3454)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.commons.dbcp.DelegatingPreparedStatement.executeUpdate(DelegatingPreparedStatement.java:105)
at org.apache.activemq.store.jdbc.adapter.DefaultJDBCAdapter.doDeleteOldMessages(DefaultJDBCAdapter.java:820)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter.cleanup(JDBCPersistenceAdapter.java:330)
at org.apache.activemq.store.jdbc.JDBCPersistenceAdapter$3.run(JDBCPersistenceAdapter.java:311)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRunAndReset(FutureTask.java:351)
at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:722)
2013-11-28 17:49:33,453 | WARN | Error while closing connection: OALL8 is in an inconsistent state, due to: OALL8 is in an inconsistent state | org.apache.activemq.store.jdbc.JDBCPersistenceAdapter | ActiveMQ Transport: tcp:///10.131.91.198:51143@61618
java.sql.SQLException: OALL8 is in an inconsistent state
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:112)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:146)
at oracle.jdbc.driver.DatabaseError.throwSqlException(DatabaseError.java:208)
at oracle.jdbc.driver.T4C8Oall.marshal(T4C8Oall.java:352)
at oracle.jdbc.driver.T4CPreparedStatement.doOall8(T4CPreparedStatement.java:205)