Old Broker IPs
192.168.3.252:13598 (id: 43 rack: null)
192.168.3.251:13598 (id: 45 rack: null)
192.168.3.34:13598 (id: 49 rack: null)

New Broker IPs
192.168.3.23:13598 (id: 56 rack: null)
192.168.3.253:13598 (id: 58 rack: null)
192.168.3.254:13598 (id: 59 rack: null)

Before broker upgrade:
"timestamp": "2021-10-19T15:36:01.751Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, groupId=my-streams-app] Updated cluster metadata updateVersion 8 to MetadataCache{clusterId='EE27S-enTseV-wYht0DmQQ', nodes={49=192.168.3.34:13598 (id: 49 rack: null), 43=192.168.3.252:13598 (id: 43 rack: null), 45=192.168.3.251:13598 (id: 45 rack: null)}, partitions=[PartitionMetadata(error=NONE, partition=engagement-4, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-8, leader=Optional[45], leaderEpoch=Optional[17], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-3, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-7, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-2, leader=Optional[45], leaderEpoch=Optional[14], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-10, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-1, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-9, leader=Optional[45], leaderEpoch=Optional[14], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-8, leader=Optional[45], leaderEpoch=Optional[16], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-7, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-11, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-6, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-5, leader=Optional[45], leaderEpoch=Optional[15], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-0, leader=Optional[43], leaderEpoch=Optional[15], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-11, leader=Optional[45], leaderEpoch=Optional[14], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-10, leader=Optional[45], leaderEpoch=Optional[16], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-2, leader=Optional[45], leaderEpoch=Optional[16], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-9, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-1, leader=Optional[43], leaderEpoch=Optional[17], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-4, leader=Optional[45], leaderEpoch=Optional[14], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-3, leader=Optional[45], leaderEpoch=Optional[16], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-6, leader=Optional[45], leaderEpoch=Optional[16], replicas=45,43, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-5, leader=Optional[43], leaderEpoch=Optional[14], replicas=43,45, isr=43,45, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-0, leader=Optional[45], leaderEpoch=Optional[14], replicas=45,43, isr=43,45, offlineReplicas=)], controller=192.168.3.252:13598 (id: 43 rack: null)}","logger_name": "org.apache.kafka.clients.Metadata"

After broker upgrade:
"timestamp": "2021-10-19T16:42:29.675Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, groupId=my-streams-app] Updated cluster metadata updateVersion 48 to MetadataCache{clusterId='EE27S-enTseV-wYht0DmQQ', nodes={56=192.168.3.23:13598 (id: 56 rack: null), 58=192.168.3.253:13598 (id: 58 rack: null), 59=192.168.3.254:13598 (id: 59 rack: null)}, partitions=[PartitionMetadata(error=NONE, partition=engagement-4, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-8, leader=Optional[58], leaderEpoch=Optional[24], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-3, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-7, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-2, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-10, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-1, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-9, leader=Optional[56], leaderEpoch=Optional[21], replicas=58,56, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-8, leader=Optional[58], leaderEpoch=Optional[23], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-7, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-11, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-6, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-5, leader=Optional[56], leaderEpoch=Optional[22], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-0, leader=Optional[56], leaderEpoch=Optional[22], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-11, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-10, leader=Optional[56], leaderEpoch=Optional[23], replicas=58,56, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-2, leader=Optional[56], leaderEpoch=Optional[23], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-9, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-1, leader=Optional[58], leaderEpoch=Optional[24], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-4, leader=Optional[56], leaderEpoch=Optional[21], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-3, leader=Optional[58], leaderEpoch=Optional[23], replicas=58,56, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-6, leader=Optional[56], leaderEpoch=Optional[23], replicas=56,58, isr=56,58, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-command-5, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=), PartitionMetadata(error=NONE, partition=engagement-0, leader=Optional[58], leaderEpoch=Optional[21], replicas=58,56, isr=58,56, offlineReplicas=)], controller=192.168.3.253:13598 (id: 58 rack: null)}","logger_name": "org.apache.kafka.clients.Metadata"

Rebalance:
"timestamp": "2021-10-19T16:42:51.317Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, groupId=my-streams-app] Sending JOIN_GROUP request with header RequestHeader(apiKey=JOIN_GROUP, apiVersion=7, clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, correlationId=27317) and timeout 305000 to node 2147483588: {group_id=my-streams-app,session_timeout_ms=10000,rebalance_timeout_ms=300000,member_id=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer-62d1d7b6-4447-4755-9005-3c01e8e28fbb,group_instance_id=null,protocol_type=consumer,protocols=[{name=stream,metadata=java.nio.HeapByteBuffer[pos=0 lim=247 cap=247],_tagged_fields={}}],_tagged_fields={}}","logger_name": "org.apache.kafka.clients.NetworkClient"

Re-connection (StreamThread) succeeds:
"timestamp": "2021-10-19T16:42:51.317Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, groupId=my-streams-app] Sending JoinGroup (JoinGroupRequestData(groupId='my-streams-app', sessionTimeoutMs=10000, rebalanceTimeoutMs=300000, memberId='my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer-62d1d7b6-4447-4755-9005-3c01e8e28fbb', groupInstanceId=null, protocolType='consumer', protocols=[JoinGroupRequestProtocol(name='stream', metadata=[0, 1, 0, 0, 0, 2, 0, 10, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 0, 18, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 45, 99, 111, 109, 109, 97, 110, 100, 0, 0, 0, 113, 0, 0, 0, 8, 0, 0, 0, 8, 75, 73, -43, 2, 58, -66, 71, 24, -101, 116, -111, 4, -73, -15, 26, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 3, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 5, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 7, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 9, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 11, -1, -1, -1, -1, -1, -1, -1, -2, 5, 0, 0, 0, 2, 0, 10, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 11, 0, 0, 0, 3, 0, 18, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 45, 99, 111, 109, 109, 97, 110, 100, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 11, 0, 0, 0, 9])])) to coordinator 192.168.3.254:13598 (id: 2147483588 rack: null)","logger_name": "org.apache.kafka.clients.consumer.internals.AbstractCoordinator"
"timestamp": "2021-10-19T16:42:51.321Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, groupId=my-streams-app] Received JOIN_GROUP response from node 2147483588 for request with header RequestHeader(apiKey=JOIN_GROUP, apiVersion=7, clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer, correlationId=27317): JoinGroupResponseData(throttleTimeMs=0, errorCode=0, generationId=31, protocolType='consumer', protocolName='stream', leader='my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer-62d1d7b6-4447-4755-9005-3c01e8e28fbb', memberId='my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer-62d1d7b6-4447-4755-9005-3c01e8e28fbb', members=[JoinGroupResponseMember(memberId='my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-consumer-62d1d7b6-4447-4755-9005-3c01e8e28fbb', groupInstanceId=null, metadata=[0, 1, 0, 0, 0, 2, 0, 10, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 0, 18, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 45, 99, 111, 109, 109, 97, 110, 100, 0, 0, 0, 113, 0, 0, 0, 8, 0, 0, 0, 8, 75, 73, -43, 2, 58, -66, 71, 24, -101, 116, -111, 4, -73, -15, 26, 40, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 1, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 3, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 5, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 7, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 9, -1, -1, -1, -1, -1, -1, -1, -2, 0, 0, 0, 11, -1, -1, -1, -1, -1, -1, -1, -2, 5, 0, 0, 0, 2, 0, 10, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 11, 0, 0, 0, 3, 0, 18, 101, 110, 103, 97, 103, 101, 109, 101, 110, 116, 45, 99, 111, 109, 109, 97, 110, 100, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 11, 0, 0, 0, 9])])","logger_name": "org.apache.kafka.clients.NetworkClient"

Re-connection (restore-consumer) fails (& loops):
"timestamp": "2021-10-19T16:42:51.601Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-restore-consumer, groupId=null] Sending ListOffsetRequest ListOffsetRequestData(replicaId=-1, isolationLevel=0, topics=[ListOffsetTopic(name='my-streams-app-fsm-state-store-changelog', partitions=[ListOffsetPartition(partitionIndex=10, currentLeaderEpoch=18, timestamp=-2, maxNumOffsets=1)])]) to broker 192.168.3.34:13598 (id: 49 rack: null)","logger_name": "org.apache.kafka.clients.consumer.internals.Fetcher"
--
"timestamp": "2021-10-19T16:42:51.603Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-restore-consumer, groupId=null] Connection with 192.168.3.34/192.168.3.34 disconnected",
"logger_name": "org.apache.kafka.common.network.Selector","stack_trace": "java.io.EOFException: null\n\tat org.apache.kafka.common.network.SslTransportLayer.write(SslTransportLayer.java:715)\n\tat org.apache.kafka.common.network.SslTransportLayer.write(SslTransportLayer.java:738)\n\tat org.apache.kafka.common.network.SslTransportLayer.write(SslTransportLayer.java:763)\n\tat org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60)\n\tat org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:425)\n\tat org.apache.kafka.common.network.Selector.write(Selector.java:644)\n\tat org.apache.kafka.common.network.Selector.attemptWrite(Selector.java:637)\n\tat org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:593)\n\tat org.apache.kafka.common.network.Selector.poll(Selector.java:481)\n\tat org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:563)\n\tat org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:265)\n\tat org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:236)\n\tat org.apache.kafka.clients.consumer.internals.ConsumerNetworkClient.poll(ConsumerNetworkClient.java:227)\n\tat org.apache.kafka.clients.consumer.KafkaConsumer.position(KafkaConsumer.java:1751)\n\tat org.apache.kafka.clients.consumer.KafkaConsumer.position(KafkaConsumer.java:1709)\n\tat io.opentracing.contrib.kafka.TracingKafkaConsumer.position(TracingKafkaConsumer.java:200)\n\tat org.apache.kafka.streams.processor.internals.StoreChangelogReader.prepareChangelogs(StoreChangelogReader.java:834)\n\tat org.apache.kafka.streams.processor.internals.StoreChangelogReader.initializeChangelogs(StoreChangelogReader.java:738)\n\tat org.apache.kafka.streams.processor.internals.StoreChangelogReader.restore(StoreChangelogReader.java:414)\n\tat org.apache.kafka.streams.processor.internals.StreamThread.initializeAndRestorePhase(StreamThread.java:771)\n\tat org.apache.kafka.streams.processor.internals.StreamThread.runOnce(StreamThread.java:636)\n\tat org.apache.kafka.streams.processor.internals.StreamThread.runLoop(StreamThread.java:564)\n\tat org.apache.kafka.streams.processor.internals.StreamThread.run(StreamThread.java:523)\n"
--
"timestamp": "2021-10-19T16:42:51.761Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-restore-consumer, groupId=null] Initiating connection to node 192.168.3.252:13598 (id: 43 rack: null) using address /192.168.3.252","logger_name": "org.apache.kafka.clients.NetworkClient"
--
"timestamp": "2021-10-19T16:42:51.767Z","message": "[Consumer clientId=my-streams-app-4b49d502-3abe-4718-9b74-9104b7f11a28-StreamThread-1-restore-consumer, groupId=null] Initiating connection to node 192.168.3.251:13598 (id: 45 rack: null) using address /192.168.3.251","logger_name": "org.apache.kafka.clients.NetworkClient"