Description
To reproduce:
1. Start Kafka Connect in distributed mode without Kafka running ./bin/connect-distributed.sh config/connect-distributed.properties
2. Ctrl+C fails to terminate the process
thread dump:
Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.92-b14 mixed mode):
"Thread-1" #13 prio=5 os_prio=31 tid=0x00007fc29a18a800 nid=0x7007 waiting on condition [0x0000700003129000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007bd7d91d8> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.kafka.connect.runtime.distributed.DistributedHerder.stop(DistributedHerder.java:357)
at org.apache.kafka.connect.runtime.Connect.stop(Connect.java:71)
at org.apache.kafka.connect.runtime.Connect$ShutdownHook.run(Connect.java:93)
"SIGINT handler" #27 daemon prio=9 os_prio=31 tid=0x00007fc29aa6a000 nid=0x560f in Object.wait() [0x0000700001a61000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007bd63db38> (a org.apache.kafka.connect.runtime.Connect$ShutdownHook)
at java.lang.Thread.join(Thread.java:1245)
- locked <0x00000007bd63db38> (a org.apache.kafka.connect.runtime.Connect$ShutdownHook)
at java.lang.Thread.join(Thread.java:1319)
at java.lang.ApplicationShutdownHooks.runHooks(ApplicationShutdownHooks.java:106)
at java.lang.ApplicationShutdownHooks$1.run(ApplicationShutdownHooks.java:46)
at java.lang.Shutdown.runHooks(Shutdown.java:123)
at java.lang.Shutdown.sequence(Shutdown.java:167)
at java.lang.Shutdown.exit(Shutdown.java:212)
- locked <0x00000007b0244600> (a java.lang.Class for java.lang.Shutdown)
at java.lang.Terminator$1.handle(Terminator.java:52)
at sun.misc.Signal$1.run(Signal.java:212)
at java.lang.Thread.run(Thread.java:745)
"kafka-producer-network-thread | producer-1" #15 daemon prio=5 os_prio=31 tid=0x00007fc29a0b7000 nid=0x7a03 runnable [0x0000700002608000]
java.lang.Thread.State: RUNNABLE
at sun.nio.ch.KQueueArrayWrapper.kevent0(Native Method)
at sun.nio.ch.KQueueArrayWrapper.poll(KQueueArrayWrapper.java:198)
at sun.nio.ch.KQueueSelectorImpl.doSelect(KQueueSelectorImpl.java:117)
at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86)
- locked <0x00000007bd7788d8> (a sun.nio.ch.Util$2)
- locked <0x00000007bd7788e8> (a java.util.Collections$UnmodifiableSet)
- locked <0x00000007bd778888> (a sun.nio.ch.KQueueSelectorImpl)
at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97)
at org.apache.kafka.common.network.Selector.select(Selector.java:470)
at org.apache.kafka.common.network.Selector.poll(Selector.java:286)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:260)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:235)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:134)
at java.lang.Thread.run(Thread.java:745)
"DistributedHerder" #14 prio=5 os_prio=31 tid=0x00007fc29a11e000 nid=0x7803 waiting on condition [0x0000700002505000]
java.lang.Thread.State: TIMED_WAITING (sleeping)
at java.lang.Thread.sleep(Native Method)
at org.apache.kafka.common.utils.SystemTime.sleep(SystemTime.java:37)
at org.apache.kafka.clients.consumer.internals.Fetcher.getTopicMetadata(Fetcher.java:299)
at org.apache.kafka.clients.consumer.KafkaConsumer.partitionsFor(KafkaConsumer.java:1310)
at org.apache.kafka.connect.util.KafkaBasedLog.start(KafkaBasedLog.java:131)
at org.apache.kafka.connect.storage.KafkaOffsetBackingStore.start(KafkaOffsetBackingStore.java:86)
at org.apache.kafka.connect.runtime.Worker.start(Worker.java:115)
at org.apache.kafka.connect.runtime.AbstractHerder.startServices(AbstractHerder.java:112)
at org.apache.kafka.connect.runtime.distributed.DistributedHerder.run(DistributedHerder.java:169)
at java.lang.Thread.run(Thread.java:745)
"RMI TCP Accept-0" #12 daemon prio=5 os_prio=31 tid=0x00007fc299aed800 nid=0x740f runnable [0x00007000022ff000]
java.lang.Thread.State: RUNNABLE
at java.net.PlainSocketImpl.socketAccept(Native Method)
at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409)
at java.net.ServerSocket.implAccept(ServerSocket.java:545)
at java.net.ServerSocket.accept(ServerSocket.java:513)
at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:400)
at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:372)
at java.lang.Thread.run(Thread.java:745)
"Service Thread" #10 daemon prio=9 os_prio=31 tid=0x00007fc29a87d800 nid=0x6e03 runnable [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C1 CompilerThread3" #9 daemon prio=9 os_prio=31 tid=0x00007fc29a02c000 nid=0x6c03 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread2" #8 daemon prio=9 os_prio=31 tid=0x00007fc29a829800 nid=0x6a03 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread1" #7 daemon prio=9 os_prio=31 tid=0x00007fc298968800 nid=0x6803 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"C2 CompilerThread0" #6 daemon prio=9 os_prio=31 tid=0x00007fc29a829000 nid=0x6603 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Signal Dispatcher" #5 daemon prio=9 os_prio=31 tid=0x00007fc298966800 nid=0x6403 waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=31 tid=0x00007fc299807000 nid=0x570b waiting on condition [0x0000000000000000]
java.lang.Thread.State: RUNNABLE
"Finalizer" #3 daemon prio=8 os_prio=31 tid=0x00007fc298965000 nid=0x5003 in Object.wait() [0x000070000195e000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007b0149a60> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:143)
- locked <0x00000007b0149a60> (a java.lang.ref.ReferenceQueue$Lock)
at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:164)
at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:209)
"Reference Handler" #2 daemon prio=10 os_prio=31 tid=0x00007fc29b804000 nid=0x4e03 in Object.wait() [0x000070000185b000]
java.lang.Thread.State: WAITING (on object monitor)
at java.lang.Object.wait(Native Method)
- waiting on <0x00000007b016ddb0> (a java.lang.ref.Reference$Lock)
at java.lang.Object.wait(Object.java:502)
at java.lang.ref.Reference.tryHandlePending(Reference.java:191)
- locked <0x00000007b016ddb0> (a java.lang.ref.Reference$Lock)
at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153)
"main" #1 prio=5 os_prio=31 tid=0x00007fc29a802000 nid=0xc0f waiting on condition [0x0000700000219000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000007bd63db08> (a java.util.concurrent.CountDownLatch$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231)
at org.apache.kafka.connect.runtime.Connect.awaitStop(Connect.java:82)
at org.apache.kafka.connect.cli.ConnectDistributed.main(ConnectDistributed.java:95)
"VM Thread" os_prio=31 tid=0x00007fc29a02b000 nid=0x4c03 runnable
"Gang worker#0 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298813800 nid=0x1607 runnable
"Gang worker#1 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298814000 nid=0x2603 runnable
"Gang worker#2 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298814800 nid=0x2803 runnable
"Gang worker#3 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298815000 nid=0x2a03 runnable
"Gang worker#4 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298816000 nid=0x2c03 runnable
"Gang worker#5 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298816800 nid=0x2e03 runnable
"Gang worker#6 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298817000 nid=0x3003 runnable
"Gang worker#7 (Parallel GC Threads)" os_prio=31 tid=0x00007fc298817800 nid=0x3203 runnable
"G1 Main Concurrent Mark GC Thread" os_prio=31 tid=0x00007fc298833800 nid=0x4603 runnable
"Gang worker#0 (G1 Parallel Marking Threads)" os_prio=31 tid=0x00007fc298834800 nid=0x4803 runnable
"Gang worker#1 (G1 Parallel Marking Threads)" os_prio=31 tid=0x00007fc298835000 nid=0x4a03 runnable
"G1 Concurrent Refinement Thread#0" os_prio=31 tid=0x00007fc29881e800 nid=0x4403 runnable
"G1 Concurrent Refinement Thread#1" os_prio=31 tid=0x00007fc29881d800 nid=0x4203 runnable
"G1 Concurrent Refinement Thread#2" os_prio=31 tid=0x00007fc29881d000 nid=0x4003 runnable
"G1 Concurrent Refinement Thread#3" os_prio=31 tid=0x00007fc29881c000 nid=0x3e03 runnable
"G1 Concurrent Refinement Thread#4" os_prio=31 tid=0x00007fc29881b800 nid=0x3c03 runnable
"G1 Concurrent Refinement Thread#5" os_prio=31 tid=0x00007fc29881a800 nid=0x3a03 runnable
"G1 Concurrent Refinement Thread#6" os_prio=31 tid=0x00007fc29881a000 nid=0x3803 runnable
"G1 Concurrent Refinement Thread#7" os_prio=31 tid=0x00007fc298819000 nid=0x3603 runnable
"G1 Concurrent Refinement Thread#8" os_prio=31 tid=0x00007fc298818800 nid=0x3403 runnable
"VM Periodic Task Thread" os_prio=31 tid=0x00007fc29896d800 nid=0x7603 waiting on condition
JNI global references: 280
Heap
garbage-first heap total 262144K, used 23429K [0x00000007b0000000, 0x00000007b0100800, 0x00000007c0000000)
region size 1024K, 14 young (14336K), 5 survivors (5120K)
Metaspace used 19993K, capacity 20266K, committed 20480K, reserved 1067008K
class space used 2430K, capacity 2513K, committed 2560K, reserved 1048576K
Attachments
Issue Links
- relates to
-
KAFKA-4306 Connect workers won't shut down if brokers are not available
-
- Resolved
-