Standard Output metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/tmp/kafka-1545337740370864125: EMPTY}) [2023-11-26 15:49:47,584] WARN [QuorumController id=1000] Performing controller activation. The metadata log appears to be empty. Appending 2 bootstrap record(s) in metadata transaction at metadata.version 3.7-IV1 from bootstrap source 'test harness'. Setting the ZK migration state to NONE since this is a de-novo KRaft cluster. (org.apache.kafka.controller.QuorumController:108) [2023-11-26 15:49:52,219] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@688d032d (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:49:52,220] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@5a4aea8c (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:49:52,220] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@121ec4df (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:49:54,121] WARN [Consumer clientId=consumer-group-399, groupId=group] Error while fetching metadata with correlation id 2 : {e2etopic=TOPIC_AUTHORIZATION_FAILED} (org.apache.kafka.clients.NetworkClient:1145) [2023-11-26 15:49:54,122] ERROR [Consumer clientId=consumer-group-399, groupId=group] Topic authorization failed for topics [e2etopic] (org.apache.kafka.clients.Metadata:456) [2023-11-26 15:49:54,720] WARN [Consumer clientId=consumer-group-399, groupId=group] Error while fetching metadata with correlation id 6 : {topic2=TOPIC_AUTHORIZATION_FAILED} (org.apache.kafka.clients.NetworkClient:1145) [2023-11-26 15:49:54,721] ERROR [Consumer clientId=consumer-group-399, groupId=group] Topic authorization failed for topics [topic2] (org.apache.kafka.clients.Metadata:456) [2023-11-26 15:49:54,986] WARN [Consumer clientId=consumer-group-399, groupId=group] Close timed out with 2 pending requests to coordinator, terminating client connections (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator:1123) [2023-11-26 15:49:56,849] WARN [NodeToControllerChannelManager id=1000 name=registration] Attempting to close NetworkClient that has already been closed. (org.apache.kafka.clients.NetworkClient:667) [2023-11-26 15:49:56,851] ERROR [StandardAuthorizer 1000] Failed to complete initial ACL load process. (org.apache.kafka.metadata.authorizer.StandardAuthorizerData:96) java.util.concurrent.TimeoutException at kafka.server.metadata.AclPublisher.close(AclPublisher.scala:98) at org.apache.kafka.image.loader.MetadataLoader.closePublisher(MetadataLoader.java:568) at org.apache.kafka.image.loader.MetadataLoader.lambda$removeAndClosePublisher$7(MetadataLoader.java:528) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:49:56,960] WARN maxCnxns is not configured, using default value 0. (org.apache.zookeeper.server.ServerCnxnFactory:309) [2023-11-26 15:50:00,266] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:50:02,463] WARN [Consumer clientId=consumer-group-400, groupId=group] Error while fetching metadata with correlation id 2 : {e2etopic=TOPIC_AUTHORIZATION_FAILED} (org.apache.kafka.clients.NetworkClient:1145) [2023-11-26 15:50:02,464] ERROR [Consumer clientId=consumer-group-400, groupId=group] Topic authorization failed for topics [e2etopic] (org.apache.kafka.clients.Metadata:456) [2023-11-26 15:50:02,763] WARN [Consumer clientId=consumer-group-400, groupId=group] Error while fetching metadata with correlation id 6 : {topic2=TOPIC_AUTHORIZATION_FAILED} (org.apache.kafka.clients.NetworkClient:1145) [2023-11-26 15:50:02,764] ERROR [Consumer clientId=consumer-group-400, groupId=group] Topic authorization failed for topics [topic2] (org.apache.kafka.clients.Metadata:456) [2023-11-26 15:50:03,279] WARN [Consumer clientId=consumer-group-400, groupId=group] Close timed out with 2 pending requests to coordinator, terminating client connections (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator:1123) [2023-11-26 15:50:03,435] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=2, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={__consumer_offsets-0=PartitionData(topicId=aj_qjpMIQcqCHbKJNRPi5g, fetchOffset=3, logStartOffset=0, maxBytes=1048576, currentLeaderEpoch=Optional[0], lastFetchedEpoch=Optional[0])}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=1244265580, epoch=7), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,450] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=36, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=1, leaderEpoch=1, isr=[1, 2], partitionEpoch=1, replicas=[0, 1, 2], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=aj_qjpMIQcqCHbKJNRPi5g}, liveLeaders=(localhost:37823 (id: 1 rack: null))) to broker localhost:43759 (id: 0 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,450] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:43759) terminated during authentication. This may happen due to any of the following reasons: (1) Authentication failed due to invalid credentials with brokers older than 1.0.0, (2) Firewall blocking Kafka TLS traffic (eg it may only allow HTTPS traffic), (3) Transient network issue. (org.apache.kafka.clients.NetworkClient:807) [2023-11-26 15:50:03,450] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:43759) terminated during authentication. This may happen due to any of the following reasons: (1) Authentication failed due to invalid credentials with brokers older than 1.0.0, (2) Firewall blocking Kafka TLS traffic (eg it may only allow HTTPS traffic), (3) Transient network issue. (org.apache.kafka.clients.NetworkClient:807) [2023-11-26 15:50:03,450] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,450] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=1, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={__consumer_offsets-0=PartitionData(topicId=aj_qjpMIQcqCHbKJNRPi5g, fetchOffset=3, logStartOffset=0, maxBytes=1048576, currentLeaderEpoch=Optional[0], lastFetchedEpoch=Optional[0])}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=642843846, epoch=5), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,450] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,552] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,552] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,552] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,552] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,552] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,552] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,654] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,654] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,654] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,654] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,654] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,654] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:43759 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,656] WARN [KafkaServer id=1] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,657] WARN [KafkaServer id=2] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 15:50:03,756] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,756] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,858] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,858] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:03,960] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:03,960] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,062] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,062] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,164] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,164] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,266] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,266] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,368] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,368] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,470] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:43759) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,470] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:43759 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:43759 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,474] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=36, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=1, leaderEpoch=1, isr=[1, 2], partitionEpoch=1, replicas=[0, 1, 2], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=aj_qjpMIQcqCHbKJNRPi5g}, liveLeaders=(localhost:37823 (id: 1 rack: null))) to broker localhost:43759 (id: 0 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,601] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:44499) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,601] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44499 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44499 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,601] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:37823) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:04,602] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:37823 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:37823 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,669] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=76, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=44499, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null), UpdateMetadataBroker(id=1, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=37823, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:37823 (id: 1 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:04,670] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=110, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=44499, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null), UpdateMetadataBroker(id=1, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=37823, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:44499 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/tmp/kafka-7741325462688375651: EMPTY}) [2023-11-26 15:50:05,781] WARN [QuorumController id=1000] Performing controller activation. The metadata log appears to be empty. Appending 2 bootstrap record(s) in metadata transaction at metadata.version 3.7-IV1 from bootstrap source 'test harness'. Setting the ZK migration state to NONE since this is a de-novo KRaft cluster. (org.apache.kafka.controller.QuorumController:108) [2023-11-26 15:50:10,501] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@7f4d35ec (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:50:10,501] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@3ef90d (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:50:10,501] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@8d6730b (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:50:11,825] WARN [Consumer clientId=consumer-group-401, groupId=group] Close timed out with 2 pending requests to coordinator, terminating client connections (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator:1123) [2023-11-26 15:50:11,996] WARN [ReplicaFetcher replicaId=0, leaderId=1, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=0, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=21332142, epoch=2), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 1 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:11,996] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=1, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={__consumer_offsets-0=PartitionData(topicId=R-Pa91zNSKiM4LINmLhTlw, fetchOffset=3, logStartOffset=0, maxBytes=1048576, currentLeaderEpoch=Optional[0], lastFetchedEpoch=Optional[0])}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=1314921880, epoch=4), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:13,731] WARN [NodeToControllerChannelManager id=1000 name=registration] Attempting to close NetworkClient that has already been closed. (org.apache.kafka.clients.NetworkClient:667) [2023-11-26 15:50:13,733] ERROR [StandardAuthorizer 1000] Failed to complete initial ACL load process. (org.apache.kafka.metadata.authorizer.StandardAuthorizerData:96) java.util.concurrent.TimeoutException at kafka.server.metadata.AclPublisher.close(AclPublisher.scala:98) at org.apache.kafka.image.loader.MetadataLoader.closePublisher(MetadataLoader.java:568) at org.apache.kafka.image.loader.MetadataLoader.lambda$removeAndClosePublisher$7(MetadataLoader.java:528) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:50:13,828] WARN maxCnxns is not configured, using default value 0. (org.apache.zookeeper.server.ServerCnxnFactory:309) [2023-11-26 15:50:16,878] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:50:17,305] WARN [ReplicaFetcher replicaId=1, leaderId=2, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition e2etopic-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:50:18,974] WARN [Consumer clientId=consumer-group-402, groupId=group] Close timed out with 2 pending requests to coordinator, terminating client connections (org.apache.kafka.clients.consumer.internals.ConsumerCoordinator:1123) [2023-11-26 15:50:19,141] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=StopReplicaRequest, controllerId=0, controllerEpoch=1, brokerEpoch=76, deletePartitions=false, topicStates=StopReplicaTopicState(topicName='e2etopic', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)]),StopReplicaTopicState(topicName='__consumer_offsets', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)])) to broker localhost:34587 (id: 1 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 1 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,162] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=1, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={__consumer_offsets-0=PartitionData(topicId=dF06sPf5RSWoAStM2lm44A, fetchOffset=3, logStartOffset=0, maxBytes=1048576, currentLeaderEpoch=Optional[0], lastFetchedEpoch=Optional[0])}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=266467199, epoch=4), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,162] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Error in response for fetch request (type=FetchRequest, replicaId=2, maxWait=500, minBytes=1, maxBytes=10485760, fetchData={__consumer_offsets-0=PartitionData(topicId=dF06sPf5RSWoAStM2lm44A, fetchOffset=3, logStartOffset=0, maxBytes=1048576, currentLeaderEpoch=Optional[0], lastFetchedEpoch=Optional[0])}, isolationLevel=READ_UNCOMMITTED, removed=, replaced=, metadata=(sessionId=940282923, epoch=6), rackId=) (kafka.server.ReplicaFetcherThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.server.BrokerBlockingSender.sendRequest(BrokerBlockingSender.scala:113) at kafka.server.RemoteLeaderEndPoint.fetch(RemoteLeaderEndPoint.scala:79) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:317) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3(AbstractFetcherThread.scala:131) at kafka.server.AbstractFetcherThread.$anonfun$maybeFetch$3$adapted(AbstractFetcherThread.scala:130) at scala.Option.foreach(Option.scala:437) at kafka.server.AbstractFetcherThread.maybeFetch(AbstractFetcherThread.scala:130) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:113) at kafka.server.ReplicaFetcherThread.doWork(ReplicaFetcherThread.scala:98) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,162] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=36, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=0, leaderEpoch=1, isr=[0, 2], partitionEpoch=1, replicas=[0, 2, 1], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0), LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=2, leaderEpoch=1, isr=[2, 0], partitionEpoch=1, replicas=[2, 1, 0], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=dF06sPf5RSWoAStM2lm44A, e2etopic=iQ7rH9MyS1Wr39jYGyzWwg}, liveLeaders=(localhost:40913 (id: 2 rack: null), localhost:41025 (id: 0 rack: null))) to broker localhost:41025 (id: 0 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,244] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:34587) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,244] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34587 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34587 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,262] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:41025 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:19,265] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:41025) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,265] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:41025 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:41025 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,346] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:34587) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,346] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34587 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34587 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,364] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:41025) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,364] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:41025 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:50:19,365] WARN [KafkaServer id=2] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 15:50:19,366] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:41025) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,366] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=110, partitionStates=[LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=2, leaderEpoch=2, isr=[2], partitionEpoch=2, replicas=[2, 1, 0], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={e2etopic=iQ7rH9MyS1Wr39jYGyzWwg}, liveLeaders=(localhost:40913 (id: 2 rack: null))) to broker localhost:40913 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 2 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,367] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:41025 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:41025 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,448] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:34587) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,448] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34587 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34587 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,469] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:41025) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,469] WARN [Controller id=0, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:40913) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,469] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:41025 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:41025 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,469] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:40913 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:40913 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:50:19,550] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:34587) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:50:19,550] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34587 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34587 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.S ...[truncated 579340 chars]... ronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:39,149] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=110, partitionStates=[LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=2, leaderEpoch=2, isr=[2], partitionEpoch=2, replicas=[0, 2, 1], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={e2etopic=HFWASzjpQB6wjkDr1NLwmw}, liveLeaders=(localhost:44921 (id: 2 rack: null))) to broker localhost:44921 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:39,228] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:44921) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:39,228] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44921 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44921 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:39,242] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=110, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=44921, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:44921 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/tmp/kafka-12880388903678668730: EMPTY}) [2023-11-26 15:53:40,005] WARN [QuorumController id=1000] Performing controller activation. The metadata log appears to be empty. Appending 2 bootstrap record(s) in metadata transaction at metadata.version 3.7-IV1 from bootstrap source 'test harness'. Setting the ZK migration state to NONE since this is a de-novo KRaft cluster. (org.apache.kafka.controller.QuorumController:108) [2023-11-26 15:53:44,413] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@3c978d0a (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:53:44,413] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@edb39ea (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:53:44,413] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@a378aa5 (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:53:45,922] WARN [NodeToControllerChannelManager id=1000 name=registration] Attempting to close NetworkClient that has already been closed. (org.apache.kafka.clients.NetworkClient:667) [2023-11-26 15:53:45,924] ERROR [StandardAuthorizer 1000] Failed to complete initial ACL load process. (org.apache.kafka.metadata.authorizer.StandardAuthorizerData:96) java.util.concurrent.TimeoutException at kafka.server.metadata.AclPublisher.close(AclPublisher.scala:98) at org.apache.kafka.image.loader.MetadataLoader.closePublisher(MetadataLoader.java:568) at org.apache.kafka.image.loader.MetadataLoader.lambda$removeAndClosePublisher$7(MetadataLoader.java:528) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 15:53:46,017] WARN maxCnxns is not configured, using default value 0. (org.apache.zookeeper.server.ServerCnxnFactory:309) [2023-11-26 15:53:48,936] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:53:48,961] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:53:49,263] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition e2etopic-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:53:49,263] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition e2etopic-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 15:53:50,007] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:39045) terminated during authentication. This may happen due to any of the following reasons: (1) Authentication failed due to invalid credentials with brokers older than 1.0.0, (2) Firewall blocking Kafka TLS traffic (eg it may only allow HTTPS traffic), (3) Transient network issue. (org.apache.kafka.clients.NetworkClient:807) [2023-11-26 15:53:50,007] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:39045) terminated during authentication. This may happen due to any of the following reasons: (1) Authentication failed due to invalid credentials with brokers older than 1.0.0, (2) Firewall blocking Kafka TLS traffic (eg it may only allow HTTPS traffic), (3) Transient network issue. (org.apache.kafka.clients.NetworkClient:807) [2023-11-26 15:53:50,008] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,007] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,110] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:39045) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,110] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:39045) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,110] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,110] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,212] WARN [KafkaServer id=1] Connection to node 0 (localhost/127.0.0.1:39045) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,212] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:39045) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,213] WARN [KafkaServer id=1] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,213] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:39045 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,216] WARN [KafkaServer id=1] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,216] WARN [KafkaServer id=2] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 15:53:50,625] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,625] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,625] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,625] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,726] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,727] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,726] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,727] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,828] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,828] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,828] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,828] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,930] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,930] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:50,930] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:50,931] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,032] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:51,032] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,033] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:51,033] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,133] WARN [Controller id=2, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:44655) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:51,133] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:44655 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:44655 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,134] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:32777) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 15:53:51,134] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:32777 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:32777 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,144] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=76, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=32777, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null), UpdateMetadataBroker(id=1, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=44655, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:44655 (id: 1 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 15:53:51,144] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=110, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=32777, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null), UpdateMetadataBroker(id=1, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=44655, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:32777 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) metaPropertiesEnsemble=MetaPropertiesEnsemble(metadataLogDir=Optional.empty, dirs={/tmp/kafka-3178058691283215093: EMPTY}) [2023-11-26 17:16:03,091] WARN [QuorumController id=1000] Performing controller activation. The metadata log appears to be empty. Appending 2 bootstrap record(s) in metadata transaction at metadata.version 3.7-IV1 from bootstrap source 'test harness'. Setting the ZK migration state to NONE since this is a de-novo KRaft cluster. (org.apache.kafka.controller.QuorumController:108) [2023-11-26 17:16:10,564] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@33db2dc7 (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 17:16:10,565] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@3949fba0 (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 17:16:10,564] ERROR Unexpected error handling org.apache.kafka.server.AssignmentsManager$DispatchEvent@4fdb7439 (org.apache.kafka.server.AssignmentsManager:117) java.lang.IllegalStateException: Cannot enqueue a request if the request thread is not running at kafka.server.NodeToControllerRequestThread.enqueue(NodeToControllerChannelManager.scala:309) at kafka.server.NodeToControllerChannelManagerImpl.sendRequest(NodeToControllerChannelManager.scala:239) at org.apache.kafka.server.AssignmentsManager$DispatchEvent.run(AssignmentsManager.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 17:16:12,326] WARN [NodeToControllerChannelManager id=1000 name=registration] Attempting to close NetworkClient that has already been closed. (org.apache.kafka.clients.NetworkClient:667) [2023-11-26 17:16:12,329] ERROR [StandardAuthorizer 1000] Failed to complete initial ACL load process. (org.apache.kafka.metadata.authorizer.StandardAuthorizerData:96) java.util.concurrent.TimeoutException at kafka.server.metadata.AclPublisher.close(AclPublisher.scala:98) at org.apache.kafka.image.loader.MetadataLoader.closePublisher(MetadataLoader.java:568) at org.apache.kafka.image.loader.MetadataLoader.lambda$removeAndClosePublisher$7(MetadataLoader.java:528) at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210) at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181) at java.base/java.lang.Thread.run(Thread.java:833) [2023-11-26 17:16:12,449] WARN maxCnxns is not configured, using default value 0. (org.apache.zookeeper.server.ServerCnxnFactory:309) [2023-11-26 17:16:16,173] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 17:16:16,182] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition __consumer_offsets-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 17:16:16,527] WARN [ReplicaFetcher replicaId=2, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition e2etopic-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 17:16:16,527] WARN [ReplicaFetcher replicaId=1, leaderId=0, fetcherId=0] Received UNKNOWN_TOPIC_ID from the leader for partition e2etopic-0. This error may be returned transiently when the partition is being created or deleted, but it is not expected to persist. (kafka.server.ReplicaFetcherThread:70) [2023-11-26 17:16:17,509] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=StopReplicaRequest, controllerId=0, controllerEpoch=1, brokerEpoch=76, deletePartitions=false, topicStates=StopReplicaTopicState(topicName='e2etopic', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)]),StopReplicaTopicState(topicName='__consumer_offsets', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)])) to broker localhost:33411 (id: 1 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 1 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,525] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=36, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=0, leaderEpoch=1, isr=[0, 2], partitionEpoch=1, replicas=[0, 2, 1], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0), LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=0, leaderEpoch=1, isr=[0, 2], partitionEpoch=1, replicas=[0, 1, 2], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=FdMBIlUXTGWwl6HmjQYaBg, e2etopic=fRL-rEfERheF5KNGHbyTBw}, liveLeaders=(localhost:34771 (id: 0 rack: null))) to broker localhost:34771 (id: 0 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to 0 was disconnected before the response was read at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:100) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:258) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,528] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to 0 was disconnected before the response was read (kafka.server.KafkaServer:70) [2023-11-26 17:16:17,613] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:33411) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,614] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:33411 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:33411 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,629] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,629] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,633] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,634] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:34771 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 17:16:17,717] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:33411) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,718] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:33411 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:33411 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,732] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,732] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,737] WARN [KafkaServer id=2] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,737] WARN [KafkaServer id=2] Error during controlled shutdown, possibly because leader movement took longer than the configured controller.socket.timeout.ms and/or request.timeout.ms: Connection to localhost:34771 (id: 0 rack: null) failed. (kafka.server.KafkaServer:70) [2023-11-26 17:16:17,742] WARN [KafkaServer id=2] Proceeding to do an unclean shutdown as all the controlled shutdown attempts failed (kafka.server.KafkaServer:70) [2023-11-26 17:16:17,822] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:33411) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,822] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:33411 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:33411 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,835] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,835] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,925] WARN [Controller id=0, targetBrokerId=1] Connection to node 1 (localhost/127.0.0.1:33411) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,926] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:33411 (id: 1 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:33411 (id: 1 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:17,938] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:17,939] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,006] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=StopReplicaRequest, controllerId=0, controllerEpoch=1, brokerEpoch=76, deletePartitions=false, topicStates=StopReplicaTopicState(topicName='e2etopic', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)]),StopReplicaTopicState(topicName='__consumer_offsets', partitionStates=[StopReplicaPartitionState(partitionIndex=0, leaderEpoch=0, deletePartition=false)])) to broker localhost:33411 (id: 1 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,042] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:18,043] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,051] WARN [Controller id=0, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:40075) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:18,052] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:40075 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:40075 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,146] WARN [Controller id=0, targetBrokerId=0] Connection to node 0 (localhost/127.0.0.1:34771) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:18,147] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:34771 (id: 0 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:34771 (id: 0 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,158] WARN [Controller id=0, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:40075) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:18,158] WARN [RequestSendThread controllerId=0] Controller 0's connection to broker localhost:40075 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:40075 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,194] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=36, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=0, leaderEpoch=1, isr=[0, 2], partitionEpoch=1, replicas=[0, 2, 1], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0), LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=0, leaderEpoch=1, isr=[0, 2], partitionEpoch=1, replicas=[0, 1, 2], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=FdMBIlUXTGWwl6HmjQYaBg, e2etopic=fRL-rEfERheF5KNGHbyTBw}, liveLeaders=(localhost:34771 (id: 0 rack: null))) to broker localhost:34771 (id: 0 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,197] WARN [RequestSendThread controllerId=0] Controller 0 epoch 1 fails to send request (type=LeaderAndIsRequest, controllerId=0, controllerEpoch=1, brokerEpoch=110, partitionStates=[LeaderAndIsrPartitionState(topicName='__consumer_offsets', partitionIndex=0, controllerEpoch=1, leader=2, leaderEpoch=2, isr=[2], partitionEpoch=2, replicas=[0, 2, 1], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0), LeaderAndIsrPartitionState(topicName='e2etopic', partitionIndex=0, controllerEpoch=1, leader=2, leaderEpoch=2, isr=[2], partitionEpoch=2, replicas=[0, 1, 2], addingReplicas=[], removingReplicas=[], isNew=false, leaderRecoveryState=0)], topicIds={__consumer_offsets=FdMBIlUXTGWwl6HmjQYaBg, e2etopic=fRL-rEfERheF5KNGHbyTBw}, liveLeaders=(localhost:40075 (id: 2 rack: null))) to broker localhost:40075 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,320] WARN [Controller id=2, targetBrokerId=2] Connection to node 2 (localhost/127.0.0.1:40075) could not be established. Node may not be available. (org.apache.kafka.clients.NetworkClient:814) [2023-11-26 17:16:18,321] WARN [RequestSendThread controllerId=2] Controller 2's connection to broker localhost:40075 (id: 2 rack: null) was unsuccessful (kafka.controller.RequestSendThread:72) java.io.IOException: Connection to localhost:40075 (id: 2 rack: null) failed. at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71) at kafka.controller.RequestSendThread.brokerReady(ControllerChannelManager.scala:298) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:251) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131) [2023-11-26 17:16:18,338] WARN [RequestSendThread controllerId=2] Controller 2 epoch 2 fails to send request (type: UpdateMetadataRequest=, controllerId=2, controllerEpoch=2, kraftController=false, type=UNKNOWN, brokerEpoch=110, partitionStates=[], liveBrokers=UpdateMetadataBroker(id=2, v0Host='', v0Port=0, endpoints=[UpdateMetadataEndpoint(port=40075, host='localhost', listener='SASL_SSL', securityProtocol=3)], rack=null)) to broker localhost:40075 (id: 2 rack: null). Reconnecting to broker. (kafka.controller.RequestSendThread:72) java.lang.InterruptedException at java.base/java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1081) at java.base/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:276) at org.apache.kafka.server.util.ShutdownableThread.pause(ShutdownableThread.java:117) at kafka.controller.RequestSendThread.doWork(ControllerChannelManager.scala:239) at org.apache.kafka.server.util.ShutdownableThread.run(ShutdownableThread.java:131)