[CASSANDRA-12238] Unexpected exception during request java.lang.IndexOutOfBoundsException: null

• Type: Bug
• Status: Open
• Priority: Normal
• Resolution: Unresolved
• Fix Version/s: 3.0.x, 3.11.x, 5.x
• Component/s: Legacy/Core
• Labels: counters remove-reopen

Environment

datastax-ddc-3.7.0, CentOS 7, x86, JDK 1.8.0u91

• Severity: Normal
• Assignee: Unassigned
• Reporter: Gábor Auth
• Created: 19/Jul/16 16:25
• Updated: 07/Mar/23 10:54

Description

Our two DCs and eight nodes cluster throws sometimes:

ERROR [SharedPool-Worker-25] 2016-07-19 16:22:14,643 ErrorMessage.java:338 - Unexpected exception during request
java.lang.IndexOutOfBoundsException: null
        at java.nio.Buffer.checkIndex(Buffer.java:546) ~[na:1.8.0_91]
        at java.nio.HeapByteBuffer.getShort(HeapByteBuffer.java:314) ~[na:1.8.0_91]
        at org.apache.cassandra.db.context.CounterContext.headerLength(CounterContext.java:141) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.context.CounterContext.access$100(CounterContext.java:76) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.context.CounterContext$ContextState.<init>(CounterContext.java:758) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.context.CounterContext$ContextState.wrap(CounterContext.java:765) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.context.CounterContext.merge(CounterContext.java:271) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.Conflicts.mergeCounterValues(Conflicts.java:76) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.Cells.reconcile(Cells.java:143) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.Row$Merger$ColumnDataReducer.getReduced(Row.java:614) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.Row$Merger$ColumnDataReducer.getReduced(Row.java:572) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.MergeIterator$ManyToOne.consume(MergeIterator.java:217) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.MergeIterator$ManyToOne.computeNext(MergeIterator.java:156) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.Row$Merger.merge(Row.java:549) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator$MergeReducer.getReduced(UnfilteredRowIterators.java:473) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator$MergeReducer.getReduced(UnfilteredRowIterators.java:437) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.MergeIterator$ManyToOne.consume(MergeIterator.java:217) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.MergeIterator$ManyToOne.computeNext(MergeIterator.java:156) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:419) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.rows.UnfilteredRowIterators$UnfilteredRowMergeIterator.computeNext(UnfilteredRowIterators.java:279) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.utils.AbstractIterator.hasNext(AbstractIterator.java:47) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.BaseRows.hasNext(BaseRows.java:112) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.FilteredRows.isEmpty(FilteredRows.java:30) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.Filter.closeIfEmpty(Filter.java:49) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.Filter.applyToPartition(Filter.java:22) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.Filter.applyToPartition(Filter.java:6) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.db.transform.BasePartitions.hasNext(BasePartitions.java:76) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.statements.SelectStatement.process(SelectStatement.java:735) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.statements.SelectStatement.processResults(SelectStatement.java:410) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:363) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:237) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.statements.SelectStatement.execute(SelectStatement.java:78) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.QueryProcessor.processStatement(QueryProcessor.java:208) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:486) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.cql3.QueryProcessor.processPrepared(QueryProcessor.java:463) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.transport.messages.ExecuteMessage.execute(ExecuteMessage.java:130) ~[apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:507) [apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.transport.Message$Dispatcher.channelRead0(Message.java:401) [apache-cassandra-3.7.0.jar:3.7.0]
        at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) [netty-all-4.0.36.Final.jar:4.0.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:292) [netty-all-4.0.36.Final.jar:4.0.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext.access$600(AbstractChannelHandlerContext.java:32) [netty-all-4.0.36.Final.jar:4.0.36.Final]
        at io.netty.channel.AbstractChannelHandlerContext$7.run(AbstractChannelHandlerContext.java:283) [netty-all-4.0.36.Final.jar:4.0.36.Final]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [na:1.8.0_91]
        at org.apache.cassandra.concurrent.AbstractLocalAwareExecutorService$FutureTask.run(AbstractLocalAwareExecutorService.java:164) [apache-cassandra-3.7.0.jar:3.7.0]
        at org.apache.cassandra.concurrent.SEPWorker.run(SEPWorker.java:105) [apache-cassandra-3.7.0.jar:3.7.0]
        at java.lang.Thread.run(Thread.java:745) [na:1.8.0_91]