Details
Major Description
Steps to reproduce:
1. Start a 3-node cluster (`node-1`, `node-2`, `node-3`)
2. Run some load on it (read and write)
3. Shutdown a single node, wait some time, start it again
4. After a little time (say, 30 seconds) shutdown the same node again, wait some time, start it again
In my case, an issue was reproduced on commit `d593e6487aa36a806bab075d19c8b831ca096a28` (2024-04-24 IGNITE-2124) on the 2nd restart of `node-3`.
Expected results: node starts without error.
Actual results: NPE in `ignite3-startup.log`.
Error when starting the node: org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe Unable to start [node=node-3] java.util.concurrent.ExecutionException: org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe Unable to start [node=node-3] at java.base/java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:395) at java.base/java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1999) at org.apache.ignite.internal.app.IgniteRunner.main(IgniteRunner.java:74) Caused by: org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe Unable to start [node=node-3] at org.apache.ignite.internal.app.IgnitionImpl.handleStartException(IgnitionImpl.java:224) at org.apache.ignite.internal.app.IgnitionImpl.lambda$doStart$1(IgnitionImpl.java:205) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.postFire(CompletableFuture.java:610) at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:840) at java.base/java.util.concurrent.CompletableFuture$Completion.exec(CompletableFuture.java:479) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183) Caused by: java.util.concurrent.CompletionException: org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe Unable to start [node=node-3] at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:314) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:319) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:932) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture$Completion.run(CompletableFuture.java:478) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) Caused by: org.apache.ignite.lang.IgniteException: IGN-CMN-65535 TraceId:c7b27181-0977-4925-b0ba-fa07d8a3bebe Unable to start [node=node-3] at org.apache.ignite.internal.app.IgniteImpl.handleStartException(IgniteImpl.java:1185) at org.apache.ignite.internal.app.IgniteImpl.lambda$start$27(IgniteImpl.java:1129) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ... 5 more Caused by: java.util.concurrent.CompletionException: java.lang.NullPointerException at java.base/java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:314) at java.base/java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:319) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:645) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) at org.apache.ignite.internal.util.IgniteUtils.lambda$copyStateTo$5(IgniteUtils.java:1211) at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) at java.base/java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) at java.base/java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) at org.apache.ignite.internal.causality.BaseVersionedValue.copyState(BaseVersionedValue.java:315) at org.apache.ignite.internal.causality.BaseVersionedValue.complete(BaseVersionedValue.java:201) at org.apache.ignite.internal.causality.IncrementalVersionedValue.lambda$completeInternal$2(IncrementalVersionedValue.java:256) at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) at java.base/java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:883) at java.base/java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2251) at org.apache.ignite.internal.causality.IncrementalVersionedValue.completeInternal(IncrementalVersionedValue.java:256) at org.apache.ignite.internal.metastorage.server.WatchProcessor.notifyUpdateRevisionListeners(WatchProcessor.java:367) at org.apache.ignite.internal.metastorage.server.persistence.RocksDbKeyValueStorage.notifyRevisionUpdateListenerOnStart(RocksDbKeyValueStorage.java:1670) at java.base/java.util.concurrent.CompletableFuture.uniComposeStage(CompletableFuture.java:1106) at java.base/java.util.concurrent.CompletableFuture.thenCompose(CompletableFuture.java:2235) at org.apache.ignite.internal.metastorage.impl.MetaStorageManagerImpl.notifyRevisionUpdateListenerOnStart(MetaStorageManagerImpl.java:886) at org.apache.ignite.internal.app.IgniteImpl.recoverComponentsStateOnStart(IgniteImpl.java:1394) at org.apache.ignite.internal.app.IgniteImpl.lambda$start$22(IgniteImpl.java:1104) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ... 4 more Caused by: java.lang.NullPointerException at org.apache.ignite.internal.distributionzones.DistributionZonesUtil.lambda$filterDataNodes$4(DistributionZonesUtil.java:547) at java.base/java.util.stream.ReferencePipeline$2$1.accept(ReferencePipeline.java:176) at java.base/java.util.HashMap$KeySpliterator.forEachRemaining(HashMap.java:1621) at java.base/java.util.stream.AbstractPipeline.copyInto(AbstractPipeline.java:484) at java.base/java.util.stream.AbstractPipeline.wrapAndCopyInto(AbstractPipeline.java:474) at java.base/java.util.stream.ReduceOps$ReduceOp.evaluateSequential(ReduceOps.java:913) at java.base/java.util.stream.AbstractPipeline.evaluate(AbstractPipeline.java:234) at java.base/java.util.stream.ReferencePipeline.collect(ReferencePipeline.java:578) at org.apache.ignite.internal.distributionzones.DistributionZonesUtil.filterDataNodes(DistributionZonesUtil.java:550) at org.apache.ignite.internal.distributionzones.causalitydatanodes.CausalityDataNodesEngine.lambda$dataNodes$3(CausalityDataNodesEngine.java:256) at org.apache.ignite.internal.util.IgniteUtils.inBusyLock(IgniteUtils.java:831) at org.apache.ignite.internal.distributionzones.causalitydatanodes.CausalityDataNodesEngine.lambda$dataNodes$4(CausalityDataNodesEngine.java:157) at java.base/java.util.concurrent.CompletableFuture$UniApply.tryFire(CompletableFuture.java:642) ... 25 more