java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:72a5da42-c429-447a-aaf3-86c1137994dd The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.base/java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) at java.base/java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) at java.base/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$5(InternalTableImpl.java:532) at java.base/java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) at java.base/java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) at java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base/java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) at org.apache.ignite.internal.replicator.ReplicaService.lambda$sendToReplica$3(ReplicaService.java:174) at java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) at java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) at java.base/java.util.concurrent.CompletableFuture$Completion.exec(CompletableFuture.java:479) at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183) Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:72a5da42-c429-447a-aaf3-86c1137994dd The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at app//org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) at java.base@11.0.17/java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) at java.base@11.0.17/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base@11.0.17/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) at app//org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) at java.base@11.0.17/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) at java.base@11.0.17/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) at java.base@11.0.17/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) at java.base@11.0.17/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) at app//org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) at app//org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) at app//org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) at app//org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) at app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) at app//org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) at app//org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) at app//org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base@11.0.17/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base@11.0.17/java.lang.Thread.run(Thread.java:834) ------- Stdout: ------- [2023-09-06T05:35:29,963][INFO ][Test worker][ItServerDataStreamerTest] >>> Starting test: ItServerDataStreamerTest#testManyItems, displayName: testManyItems(), workDir: /opt/buildagent/work/1f60a66cd94efdef/gridgain-9/modules/runner/build/work/ItServerDataStreamerTest/static_19158670530628301 [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_1, force=false [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Waiting for actual storage state, group=1_part_1 [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_0, force=false [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Proposing lease redirection, proposed node=ClusterNodeImpl [id=d395eefd-d0c9-4924-94c7-b5cd8e47d345, name=isdst_n_1, address=172.120.5.2:3345, nodeMetadata=org.apache.ignite.network.NodeMetadata@2fdfe519] [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_3, force=false [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Waiting for actual storage state, group=1_part_3 [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_2, force=false [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Proposing lease redirection, proposed node=ClusterNodeImpl [id=d395eefd-d0c9-4924-94c7-b5cd8e47d345, name=isdst_n_1, address=172.120.5.2:3345, nodeMetadata=org.apache.ignite.network.NodeMetadata@2fdfe519] [2023-09-06T05:35:30,003][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_7, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Proposing lease redirection, proposed node=ClusterNodeImpl [id=d395eefd-d0c9-4924-94c7-b5cd8e47d345, name=isdst_n_1, address=172.120.5.2:3345, nodeMetadata=org.apache.ignite.network.NodeMetadata@2fdfe519] [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_6, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Proposing lease redirection, proposed node=ClusterNodeImpl [id=38af5dcc-c254-42ac-b9f7-743845a4076d, name=isdst_n_2, address=172.120.5.2:3346, nodeMetadata=org.apache.ignite.network.NodeMetadata@2fdfe538] [2023-09-06T05:35:30,004][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_5, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_9, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Waiting for actual storage state, group=1_part_9 [2023-09-06T05:35:30,004][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Proposing lease redirection, proposed node=ClusterNodeImpl [id=38af5dcc-c254-42ac-b9f7-743845a4076d, name=isdst_n_2, address=172.120.5.2:3346, nodeMetadata=org.apache.ignite.network.NodeMetadata@2fdfe538] [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_8, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-metastorage-_stripe_0-0][ReplicaManager] Waiting for actual storage state, group=1_part_8 [2023-09-06T05:35:30,004][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_4, force=false [2023-09-06T05:35:30,004][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_4 [2023-09-06T05:35:30,005][INFO ][%isdst_n_0%JRaft-AppendEntries-Processor-11][ReplicaManager] Lease accepted, group=1_part_8, leaseStartTime=HybridTimestamp [time=111016576942211092, physical=1693978530002, logical=20], leaseExpirationTime=HybridTimestamp [time=111016584806531072, physical=1693978650002, logical=0] [2023-09-06T05:35:30,012][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_0-0][ReplicaManager] Lease accepted, group=1_part_4, leaseStartTime=HybridTimestamp [time=111016576942211084, physical=1693978530002, logical=12], leaseExpirationTime=HybridTimestamp [time=111016584806531072, physical=1693978650002, logical=0] [2023-09-06T05:35:30,013][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-_stripe_6-0][ReplicaManager] Lease accepted, group=1_part_3, leaseStartTime=HybridTimestamp [time=111016576942211078, physical=1693978530002, logical=6], leaseExpirationTime=HybridTimestamp [time=111016584806531072, physical=1693978650002, logical=0] [2023-09-06T05:35:30,048][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-_stripe_4-0][ReplicaManager] Lease accepted, group=1_part_1, leaseStartTime=HybridTimestamp [time=111016576942211074, physical=1693978530002, logical=2], leaseExpirationTime=HybridTimestamp [time=111016584806531072, physical=1693978650002, logical=0] [2023-09-06T05:35:30,048][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-_stripe_4-0][ReplicaManager] Lease accepted, group=1_part_9, leaseStartTime=HybridTimestamp [time=111016576942211090, physical=1693978530002, logical=18], leaseExpirationTime=HybridTimestamp [time=111016584806531072, physical=1693978650002, logical=0] [2023-09-06T05:35:31,540][INFO ][%isdst_n_1%JRaft-ElectionTimer-7][NodeImpl] Node <1_part_9/isdst_n_1> term 1 start preVote. [2023-09-06T05:35:31,541][INFO ][%isdst_n_0%JRaft-StepDownTimer-6][Replicator] Replicator Replicator [state=Replicate, statInfo=, peerId=isdst_n_1, type=Follower] is going to quit [2023-09-06T05:35:31,546][INFO ][%isdst_n_2%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_5, force=false [2023-09-06T05:35:31,547][INFO ][%isdst_n_2%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_5 [2023-09-06T05:35:31,547][INFO ][%isdst_n_2%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_6, force=false [2023-09-06T05:35:31,547][INFO ][%isdst_n_2%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_6 [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_0, force=false [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_0 [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_2, force=false [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_2 [2023-09-06T05:35:31,548][INFO ][%isdst_n_0%JRaft-Request-Processor-23][NodeImpl] Node <1_part_9/isdst_n_0> received PreVoteRequest from isdst_n_1, term=2, currTerm=1, granted=false, requestLastLogId=LogId [index=12, term=1], lastLogId=LogId [index=13, term=1]. [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Received LeaseGrantedMessage for replica belonging to group=1_part_7, force=false [2023-09-06T05:35:31,548][INFO ][%isdst_n_1%MessagingService-inbound--0][ReplicaManager] Waiting for actual storage state, group=1_part_7 [2023-09-06T05:35:31,548][INFO ][%isdst_n_2%JRaft-AppendEntries-Processor-3][ReplicaManager] Lease accepted, group=1_part_6, leaseStartTime=HybridTimestamp [time=111016576974979080, physical=1693978530502, logical=8], leaseExpirationTime=HybridTimestamp [time=111016584839299072, physical=1693978650502, logical=0] [2023-09-06T05:35:31,549][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_5-0][StateMachineAdapter] onStopFollowing: LeaderChangeContext [leaderId=isdst_n_0, term=1, status=Status[ERAFTTIMEDOUT<10001>: Lost connection from leader isdst_n_0.]]. [2023-09-06T05:35:31,550][INFO ][%isdst_n_1%JRaft-Response-Processor-14][NodeImpl] Node <1_part_9/isdst_n_1> received PreVoteResponse from isdst_n_0, term=1, granted=false. [2023-09-06T05:35:31,562][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_4-0][ReplicaManager] Lease accepted, group=1_part_0, leaseStartTime=HybridTimestamp [time=111016576974979074, physical=1693978530502, logical=2], leaseExpirationTime=HybridTimestamp [time=111016584839299072, physical=1693978650502, logical=0] [2023-09-06T05:35:31,578][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_5-0][StateMachineAdapter] onStartFollowing: LeaderChangeContext [leaderId=isdst_n_0, term=1, status=Status[ENEWLEADER<10011>: Follower receives message from new leader with the same term.]]. [2023-09-06T05:35:31,584][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_6-0][ReplicaManager] Lease accepted, group=1_part_2, leaseStartTime=HybridTimestamp [time=111016576974979076, physical=1693978530502, logical=4], leaseExpirationTime=HybridTimestamp [time=111016584839299072, physical=1693978650502, logical=0] [2023-09-06T05:35:31,600][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-_stripe_4-0][StateMachineAdapter] onLeaderStop: status=Status[ERAFTTIMEDOUT<10001>: Majority of the group dies: 1/2]. [2023-09-06T05:35:31,601][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_3-0][ReplicaManager] Lease accepted, group=1_part_7, leaseStartTime=HybridTimestamp [time=111016576974979079, physical=1693978530502, logical=7], leaseExpirationTime=HybridTimestamp [time=111016584839299072, physical=1693978650502, logical=0] [2023-09-06T05:35:31,601][INFO ][%isdst_n_2%JRaft-FSMCaller-Disruptor-_stripe_2-0][ReplicaManager] Lease accepted, group=1_part_5, leaseStartTime=HybridTimestamp [time=111016576974979077, physical=1693978530502, logical=5], leaseExpirationTime=HybridTimestamp [time=111016584839299072, physical=1693978650502, logical=0] [2023-09-06T05:35:33,066][INFO ][%isdst_n_0%JRaft-ElectionTimer-4][NodeImpl] Node <1_part_9/isdst_n_0> term 1 start preVote. [2023-09-06T05:35:33,067][INFO ][%isdst_n_1%JRaft-Request-Processor-10][NodeImpl] Node <1_part_9/isdst_n_1> received PreVoteRequest from isdst_n_0, term=2, currTerm=1, granted=true, requestLastLogId=LogId [index=13, term=1], lastLogId=LogId [index=13, term=1]. [2023-09-06T05:35:33,068][INFO ][%isdst_n_0%JRaft-Response-Processor-16][NodeImpl] Node <1_part_9/isdst_n_0> received PreVoteResponse from isdst_n_1, term=1, granted=true. [2023-09-06T05:35:33,068][INFO ][%isdst_n_0%JRaft-Response-Processor-16][NodeImpl] Node <1_part_9/isdst_n_0> start vote and grant vote self, term=1. [2023-09-06T05:35:33,069][INFO ][%isdst_n_1%JRaft-Request-Processor-5][NodeImpl] Node <1_part_9/isdst_n_1> received RequestVoteRequest from isdst_n_0, term=2, currTerm=1. [2023-09-06T05:35:33,070][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_5-0][StateMachineAdapter] onStopFollowing: LeaderChangeContext [leaderId=isdst_n_0, term=1, status=Status[EHIGHERTERMRESPONSE<10008>: Raft node receives higher term RequestVoteRequest.]]. [2023-09-06T05:35:33,070][INFO ][%isdst_n_0%JRaft-Response-Processor-16][LocalRaftMetaStorage] Save raft meta, path=build/work/ItServerDataStreamerTest/static_19158670530628301/isdst_n_0/1_part_9_isdst_n_0_0/meta, term=2, votedFor=isdst_n_0, cost time=1 ms [2023-09-06T05:35:33,070][INFO ][%isdst_n_1%JRaft-Request-Processor-5][LocalRaftMetaStorage] Save raft meta, path=build/work/ItServerDataStreamerTest/static_19158670530628301/isdst_n_1/1_part_9_isdst_n_1_0/meta, term=2, votedFor=, cost time=1 ms [2023-09-06T05:35:33,071][INFO ][%isdst_n_1%JRaft-Request-Processor-5][LocalRaftMetaStorage] Save raft meta, path=build/work/ItServerDataStreamerTest/static_19158670530628301/isdst_n_1/1_part_9_isdst_n_1_0/meta, term=2, votedFor=isdst_n_0, cost time=1 ms [2023-09-06T05:35:33,072][INFO ][%isdst_n_0%JRaft-Response-Processor-17][NodeImpl] Node <1_part_9/isdst_n_0> become leader of group, term=2, conf=isdst_n_0,isdst_n_1, oldConf=. [2023-09-06T05:35:33,073][INFO ][%isdst_n_0%JRaft-Response-Processor-17][Replicator] Replicator=Replicator [state=Created, statInfo=, peerId=isdst_n_1, type=Follower]@isdst_n_1 is started [2023-09-06T05:35:33,073][INFO ][%isdst_n_1%JRaft-FSMCaller-Disruptor-_stripe_5-0][StateMachineAdapter] onStartFollowing: LeaderChangeContext [leaderId=isdst_n_0, term=2, status=Status[ENEWLEADER<10011>: Follower receives message from new leader with the same term.]]. [2023-09-06T05:35:33,075][INFO ][%isdst_n_0%JRaft-FSMCaller-Disruptor-_stripe_4-0][StateMachineAdapter] onLeaderStart: term=2. [2023-09-06T05:35:33,178][INFO ][Test worker][ItServerDataStreamerTest] >>> Stopping test: ItServerDataStreamerTest#testManyItems, displayName: testManyItems(), cost: 3215ms. ------- Stderr: ------- [2023-09-06T05:35:31,540][WARN ][%isdst_n_1%jvm-pause-detector-worker-][LongJvmPauseDetector] Possible too long JVM pause [duration=995ms] [2023-09-06T05:35:31,540][WARN ][%isdst_n_0%jvm-pause-detector-worker-][LongJvmPauseDetector] Possible too long JVM pause [duration=995ms] [2023-09-06T05:35:31,540][WARN ][%isdst_n_2%jvm-pause-detector-worker-][LongJvmPauseDetector] Possible too long JVM pause [duration=1032ms] [2023-09-06T05:35:31,540][WARN ][%isdst_n_0%JRaft-StepDownTimer-6][NodeImpl] Node <1_part_9/isdst_n_0> steps down when alive nodes don't satisfy quorum, term=1, deadNodes=isdst_n_1, conf=isdst_n_0,isdst_n_1. [2023-09-06T05:35:33,163][WARN ][%isdst_n_0%JRaft-Request-Processor-11][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577044316163, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:caa71a31-137e-4f0b-9d64-cf58bb336080 The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) ~[ignite-network-api-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 18 more [2023-09-06T05:35:33,165][WARN ][%isdst_n_0%MessagingService-inbound--0][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577149435905, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:b9319f8f-e414-47b5-8f39-b4b1af99e3f9 The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.onMessage(DefaultMessagingService.java:366) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.lambda$onMessage$4(DefaultMessagingService.java:348) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 13 more [2023-09-06T05:35:33,167][WARN ][%isdst_n_0%JRaft-Request-Processor-23][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577149632513, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:f7cba15f-6975-4365-9037-0d48b0f4895a The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) ~[ignite-network-api-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 18 more [2023-09-06T05:35:33,170][WARN ][%isdst_n_0%MessagingService-inbound--0][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577149763584, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:8e4d879a-6825-43d1-a6b0-eff836c2100b The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.onMessage(DefaultMessagingService.java:366) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.lambda$onMessage$4(DefaultMessagingService.java:348) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 13 more [2023-09-06T05:35:33,172][WARN ][%isdst_n_0%JRaft-Request-Processor-14][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577149960192, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:c0735766-f27c-435b-9f46-9d9708190aca The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) ~[ignite-network-api-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 18 more [2023-09-06T05:35:33,174][WARN ][%isdst_n_0%JRaft-Request-Processor-3][ReplicaManager] Failed to process replica request [request=ReadWriteMultiRowReplicaRequestImpl [binaryRowMessages=ArrayList [BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1], BinaryRowMessageImpl [binaryTuple=java.nio.HeapByteBuffer[pos=0 lim=12 cap=12], schemaVersion=1]], commitPartitionId=1_part_9, full=true, groupId=1_part_9, requestType=RW_UPSERT_ALL, term=1, timestampLong=111016577150025734, transactionId=018a68fd-16e7-0001-0000-0000eef4a2bb]] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:72a5da42-c429-447a-aaf3-86c1137994dd The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) ~[ignite-network-api-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] ... 18 more [2023-09-06T05:35:33,175][ERROR][ForkJoinPool.commonPool-worker-95][DataStreamer] Failed to send batch to partition 9: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:72a5da42-c429-447a-aaf3-86c1137994dd The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] java.util.concurrent.CompletionException: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: IGN-REP-6 TraceId:72a5da42-c429-447a-aaf3-86c1137994dd The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at java.util.concurrent.CompletableFuture.encodeRelay(CompletableFuture.java:367) ~[?:?] at java.util.concurrent.CompletableFuture.completeRelay(CompletableFuture.java:376) ~[?:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1074) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$4(InternalTableImpl.java:525) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.table.distributed.storage.InternalTableImpl.lambda$enlistWithRetry$5(InternalTableImpl.java:532) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:930) ~[?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:907) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2088) ~[?:?] at org.apache.ignite.internal.replicator.ReplicaService.lambda$sendToReplica$3(ReplicaService.java:174) ~[ignite-replicator-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture$Completion.exec(CompletableFuture.java:479) ~[?:?] at java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java:290) ~[?:?] at java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1020) ~[?:?] at java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1656) ~[?:?] at java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1594) ~[?:?] at java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:183) ~[?:?] Caused by: org.apache.ignite.internal.replicator.exception.PrimaryReplicaMissException: The primary replica has changed because the term has been changed [expectedPrimaryReplicaTerm=1, currentPrimaryReplicaTerm=2] at org.apache.ignite.internal.table.distributed.replicator.PartitionReplicaListener.lambda$ensureReplicaIsPrimary$160(PartitionReplicaListener.java:2391) ~[ignite-table-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:1072) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.internal.raft.RaftGroupServiceImpl.lambda$sendWithRetry$39(RaftGroupServiceImpl.java:542) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506) ~[?:?] at java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073) ~[?:?] at org.apache.ignite.network.DefaultMessagingService.onInvokeResponse(DefaultMessagingService.java:414) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.send0(DefaultMessagingService.java:213) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.DefaultMessagingService.respond(DefaultMessagingService.java:156) ~[ignite-network-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.network.MessagingService.respond(MessagingService.java:109) ~[ignite-network-api-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$NetworkRpcContext.sendResponse(IgniteRpcServer.java:234) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:52) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.RpcRequestProcessor.handleRequest(RpcRequestProcessor.java:29) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at org.apache.ignite.raft.jraft.rpc.impl.IgniteRpcServer$RpcMessageHandler.lambda$onReceived$0(IgniteRpcServer.java:193) ~[ignite-raft-9.0.127-SNAPSHOT.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) ~[?:?] at java.lang.Thread.run(Thread.java:834) ~[?:?]