[AMQ-6965] Failed to fill batch | org.apache.activemq.broker.region.cursors.AbstractStoreCursor | Scheduler java.io.EOFException: Chunk stream does not exist, page: 1723 is marked free

• Type: Bug
• Status: Open
• Priority: Major
• Resolution: Unresolved
• Affects Version/s: 5.14.5
• Component/s: KahaDB, Message Store

Environment

Active/Standby

RHEL 7.2 VM - 4 CPU - 8 GB RAM

50GB NFS4 shared

 

<broker xmlns="http://activemq.apache.org/schema/core" brokerName="CMDBROKER-10.203.14.145"
dataDirectory="${activemq.data}" advisorySupport="false">

<destinationPolicy>
<policyMap>
<policyEntries>
<policyEntry topic=">">
<!-- The constantPendingMessageLimitStrategy is used to prevent
slow topic consumers to block producers and affect other consumers
by limiting the number of messages that are retained
For more information, see:

http://activemq.apache.org/slow-consumer-handling.html

-->
<pendingMessageLimitStrategy>
<constantPendingMessageLimitStrategy limit="1000"/>
</pendingMessageLimitStrategy>
</policyEntry>
</policyEntries>
</policyMap>
</destinationPolicy>

<persistenceAdapter>
<kahaDB directory="${activemq.data}/kahadb" preallocationStrategy="zeros"/>
</persistenceAdapter>

<!--
The systemUsage controls the maximum amount of space the broker will
use before disabling caching and/or slowing down producers. For more information, see:
http://activemq.apache.org/producer-flow-control.html
-->
<systemUsage>
<systemUsage>
<memoryUsage>
<memoryUsage percentOfJvmHeap="70"/>
</memoryUsage>
<storeUsage>
<storeUsage limit="40 gb"/>
</storeUsage>
<tempUsage>
<tempUsage limit="8 gb"/>
</tempUsage>
</systemUsage>
</systemUsage>

<transportConnectors>
<!-- DOS protection, limit concurrent connections to 1000 and frame size to 100MB -->
<transportConnector name="openwire"
uri="tcp://0.0.0.0:61616?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="amqp"
uri="amqp://0.0.0.0:5672?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="stomp"
uri="stomp://0.0.0.0:61613?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="mqtt"
uri="mqtt://0.0.0.0:1883?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
<transportConnector name="ws"
uri="ws://0.0.0.0:61614?maximumConnections=1000&wireFormat.maxFrameSize=104857600"/>
</transportConnectors>

• Assignee: Jean-Baptiste Onofré
• Reporter: Alessandro Capomassi
• Created: 16/May/18 10:24
• Updated: 23/May/19 12:19

Description

2018-05-09 03:08:24,002 | ERROR | org.apache.activemq.broker.region.cursors.QueueStorePrefetch@b23483a:CMP_ExternalEventQueue,batchResetNeeded=false,size=916,cacheEnabled=false,maxBatchSize:400,hasSpace:true,pendingCachedIds.size:0,lastSyncCachedId:null,lastSyncCachedId-seq:null,lastAsyncCachedId:null,lastAsyncCachedId-seq:null,store=permits:10000,sd=nextSeq:8594,lastRet:MessageOrderCursor:[def:7667, low:0, high:0],pending:0 - Failed to fill batch | org.apache.activemq.broker.region.cursors.AbstractStoreCursor | ActiveMQ Broker[CMDBROKER-10.203.14.145] Scheduler
java.io.EOFException: Chunk stream does not exist, page: 1723 is marked free
at org.apache.activemq.store.kahadb.disk.page.Transaction$2.readPage(Transaction.java:470)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.page.Transaction$2.<init>(Transaction.java:447)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.page.Transaction.openInputStream(Transaction.java:444)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.page.Transaction.load(Transaction.java:420)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.page.Transaction.load(Transaction.java:377)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.index.BTreeIndex.loadNode(BTreeIndex.java:266)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.index.BTreeNode.getChild(BTreeNode.java:233)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.index.BTreeNode.getLeafNode(BTreeNode.java:684)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.index.BTreeNode.iterator(BTreeNode.java:647)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.index.BTreeIndex.iterator(BTreeIndex.java:236)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex$MessageOrderIterator.<init>(MessageDatabase.java:3739)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.MessageDatabase$MessageOrderIndex.iterator(MessageDatabase.java:3692)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore$4.execute(KahaDBStore.java:588)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.disk.page.Transaction.execute(Transaction.java:779)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.kahadb.KahaDBStore$KahaDBMessageStore.recoverNextMessages(KahaDBStore.java:582)[activemq-kahadb-store-5.14.5.jar:5.14.5]
at org.apache.activemq.store.ProxyMessageStore.recoverNextMessages(ProxyMessageStore.java:110)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.cursors.QueueStorePrefetch.doFillBatch(QueueStorePrefetch.java:120)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.fillBatch(AbstractStoreCursor.java:420)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.cursors.AbstractStoreCursor.hasNext(AbstractStoreCursor.java:202)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.cursors.StoreQueueCursor.hasNext(StoreQueueCursor.java:139)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue.doPageInForDispatch(Queue.java:1942)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue.pageInMessages(Queue.java:2170)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue.doBrowse(Queue.java:1132)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue.expireMessages(Queue.java:912)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue.access$100(Queue.java:104)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.broker.region.Queue$2.run(Queue.java:146)[activemq-broker-5.14.5.jar:5.14.5]
at org.apache.activemq.thread.SchedulerTimerTask.run(SchedulerTimerTask.java:33)[activemq-client-5.14.5.jar:5.14.5]
at java.util.TimerThread.mainLoop(Timer.java:566)[:1.8.0]
at java.util.TimerThread.run(Timer.java:516)[:1.8.0]